AgentFeedbackV3Test.php 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. <?php
  2. use App\Models\AgentEvent;
  3. use App\Models\AgentFeedback;
  4. use App\Models\AgentRun;
  5. use App\Models\AgentThread;
  6. use App\Models\UserInfo;
  7. use Illuminate\Foundation\Testing\RefreshDatabase;
  8. use Illuminate\Support\Str;
  9. uses(RefreshDatabase::class);
  10. /**
  11. * v3 AI 回答评价、隐式信号、自动质量指标与分析导出:
  12. *
  13. * POST /api/v3/agent-feedback 评价(可选登录,游客带 device_id)
  14. * DELETE /api/v3/agent-feedback/{message} 撤回
  15. * POST /api/v3/agent-events 隐式信号
  16. * GET /api/v3/admin/agent-feedback 管理员审阅
  17. * GET /api/v3/admin/agent-runs/digest 分析用精简导出
  18. *
  19. * 这里断言的是形状而非契约(spectator 尚未安装,见 v3-resource skill 第 3 步)。
  20. */
  21. function feedbackAdmin(): array
  22. {
  23. $uid = makeStudio('fa-'.Str::random(5));
  24. UserInfo::where('userid', $uid)->update(['role' => json_encode(['administrator'])]);
  25. return authHeader($uid);
  26. }
  27. /* ------------------------------------------------------------------ 评价 */
  28. it('lets a guest rate an answer and change the rating', function () {
  29. $device = (string) Str::uuid();
  30. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'lc_run--abc', 'rating' => 1, 'tags' => ['clear'], 'device_id' => $device])
  31. ->assertCreated()
  32. ->assertJsonPath('data.rating', 1)
  33. ->assertJsonPath('data.tags', ['clear']);
  34. $this->postJson('/api/v3/agent-feedback', [
  35. 'message_id' => 'lc_run--abc', 'rating' => -1, 'tags' => ['missed', 'other'], 'comment' => '中部有这段', 'device_id' => $device,
  36. ])->assertOk()->assertJsonPath('data.comment', '中部有这段');
  37. expect(AgentFeedback::count())->toBe(1)
  38. ->and(AgentFeedback::first()->account)->toBe('guest:'.$device);
  39. });
  40. it('keeps one rating per user and per guest device', function () {
  41. $user = (string) Str::uuid();
  42. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm1', 'rating' => 1], authHeader($user))->assertCreated();
  43. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm1', 'rating' => -1, 'device_id' => (string) Str::uuid()])->assertCreated();
  44. expect(AgentFeedback::where('message_id', 'm1')->count())->toBe(2);
  45. });
  46. it('drops the comment unless other is chosen', function () {
  47. $this->postJson('/api/v3/agent-feedback', [
  48. 'message_id' => 'm2', 'rating' => -1, 'tags' => ['slow'], 'comment' => 'x', 'device_id' => (string) Str::uuid(),
  49. ])->assertCreated()->assertJsonPath('data.comment', null);
  50. });
  51. it('rejects tags that do not match the rating', function () {
  52. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm3', 'rating' => 1, 'tags' => ['slow'], 'device_id' => (string) Str::uuid()])
  53. ->assertUnprocessable()->assertJsonValidationErrors('tags');
  54. });
  55. it('rejects an unknown tag and a bad message id', function () {
  56. $device = (string) Str::uuid();
  57. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm4', 'rating' => -1, 'tags' => ['boring'], 'device_id' => $device])
  58. ->assertUnprocessable();
  59. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'a b', 'rating' => 1, 'device_id' => $device])
  60. ->assertUnprocessable()->assertJsonValidationErrors('message_id');
  61. });
  62. it('requires a device id from guests', function () {
  63. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm5', 'rating' => 1])
  64. ->assertUnprocessable()->assertJsonValidationErrors('device_id');
  65. });
  66. it('remembers that a rating was asked for', function () {
  67. $device = (string) Str::uuid();
  68. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm6', 'rating' => 1, 'asked' => true, 'device_id' => $device]);
  69. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm6', 'rating' => -1, 'device_id' => $device])
  70. ->assertJsonPath('data.asked', true);
  71. });
  72. it('retracts a rating idempotently', function () {
  73. $device = (string) Str::uuid();
  74. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'lc_run--x', 'rating' => 1, 'device_id' => $device]);
  75. $this->deleteJson("/api/v3/agent-feedback/lc_run--x?device_id={$device}")->assertNoContent();
  76. $this->deleteJson("/api/v3/agent-feedback/lc_run--x?device_id={$device}")->assertNoContent();
  77. expect(AgentFeedback::count())->toBe(0);
  78. });
  79. it('cannot retract someone else\'s rating', function () {
  80. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm7', 'rating' => 1, 'device_id' => (string) Str::uuid()]);
  81. $this->deleteJson('/api/v3/agent-feedback/m7?device_id='.Str::uuid())->assertNoContent();
  82. expect(AgentFeedback::count())->toBe(1);
  83. });
  84. /* ------------------------------------------------------------------ 隐式信号 */
  85. it('records implicit signals', function () {
  86. $this->postJson('/api/v3/agent-events', ['message_id' => 'm8', 'kind' => 'copy', 'device_id' => (string) Str::uuid(), 'client' => 'web'])
  87. ->assertCreated()->assertJsonPath('data.kind', 'copy');
  88. $this->postJson('/api/v3/agent-events', ['message_id' => 'm8', 'kind' => 'citation_open'], authHeader((string) Str::uuid()))
  89. ->assertCreated();
  90. expect(AgentEvent::count())->toBe(2);
  91. });
  92. it('rejects an unknown signal', function () {
  93. $this->postJson('/api/v3/agent-events', ['message_id' => 'm9', 'kind' => 'like', 'device_id' => (string) Str::uuid()])
  94. ->assertUnprocessable();
  95. });
  96. /* ------------------------------------------------------------------ 评价即同意用于改进 */
  97. it('turns improve on for the rated run of the same account only', function () {
  98. $device = (string) Str::uuid();
  99. $mine = AgentThread::factory()->create(['account' => 'guest:'.$device, 'user_uid' => null, 'device_id' => $device]);
  100. $run = AgentRun::factory()->for($mine, 'thread')->create(['assistant_message_id' => 'msg-c1', 'improve' => false]);
  101. $other = AgentRun::factory()->for(AgentThread::factory(), 'thread')->create(['assistant_message_id' => 'msg-c2', 'improve' => false]);
  102. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-c1', 'rating' => 1, 'device_id' => $device])->assertCreated();
  103. // 别人的回答(消息 id 碰巧一样也不行)不受影响
  104. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-c2', 'rating' => -1, 'device_id' => $device])->assertCreated();
  105. expect($run->fresh()->improve)->toBeTrue()
  106. ->and($other->fresh()->improve)->toBeFalse();
  107. });
  108. it('marks a trace improve when the rating arrived first', function () {
  109. config(['mint.app.agent_service_key' => 'k']);
  110. $device = (string) Str::uuid();
  111. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-early', 'rating' => 1, 'device_id' => $device])->assertCreated();
  112. $run = (string) Str::uuid();
  113. $this->postJson('/api/v3/agent-traces', [
  114. 'run_id' => $run, 'thread_id' => (string) Str::uuid(), 'device_id' => $device, 'improve' => false,
  115. 'question' => '什么是慈?', 'status' => 'ok', 'answer' => '……', 'assistant_message_id' => 'msg-early',
  116. ], ['X-Agent-Service-Key' => 'k'])->assertCreated();
  117. expect(AgentRun::where('run_id', $run)->value('improve'))->toBeTrue();
  118. });
  119. /* ------------------------------------------------------------------ 管理员 */
  120. it('lists feedback with the run summary for admins only', function () {
  121. $thread = AgentThread::factory()->create();
  122. AgentRun::factory()->for($thread, 'thread')->create(['assistant_message_id' => 'msg-1', 'question' => '什么是慈?']);
  123. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-1', 'rating' => -1, 'tags' => ['missed'], 'device_id' => (string) Str::uuid()]);
  124. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-unknown', 'rating' => 1, 'device_id' => (string) Str::uuid()]);
  125. $this->getJson('/api/v3/admin/agent-feedback')->assertUnauthorized();
  126. $this->getJson('/api/v3/admin/agent-feedback', authHeader(makeStudio('np-'.Str::random(4))))->assertForbidden();
  127. $admin = feedbackAdmin();
  128. $this->getJson('/api/v3/admin/agent-feedback?filter[rating]=-1', $admin)
  129. ->assertOk()
  130. ->assertJsonCount(1, 'data')
  131. ->assertJsonPath('data.0.question', '什么是慈?')
  132. ->assertJsonPath('data.0.account_type', 'guest')
  133. ->assertJsonMissingPath('data.0.account');
  134. $this->getJson('/api/v3/admin/agent-feedback?filter[tag]=missed', $admin)->assertJsonCount(1, 'data');
  135. $this->getJson('/api/v3/admin/agent-feedback', $admin)->assertJsonPath('meta.total', 2);
  136. });
  137. it('adds feedback to the agent stats', function () {
  138. $thread = AgentThread::factory()->create();
  139. AgentRun::factory()->for($thread, 'thread')->create([
  140. 'assistant_message_id' => 'msg-s', 'started_at' => '2026-10-20 10:00:00', 'route' => 'question',
  141. ]);
  142. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'msg-s', 'rating' => -1, 'tags' => ['citation_wrong', 'slow'], 'asked' => true, 'device_id' => (string) Str::uuid()]);
  143. $data = $this->getJson('/api/v3/admin/agent-stats?from=2026-10-20&to=2026-10-20', feedbackAdmin())->json('data');
  144. expect($data['summary'])->toMatchArray(['feedback_down' => 1, 'feedback_asked' => 1])
  145. ->and($data['summary'])->not->toHaveKey('quality_assessed')
  146. ->and($data)->not->toHaveKey('quality_by_route')
  147. ->and(collect($data['feedback_reasons'])->pluck('reason')->all())->toContain('citation_wrong', 'slow')
  148. ->and($data['daily'][0]['feedback_down'])->toBe(1);
  149. });
  150. it('exports a digest without identities or opted-out runs', function () {
  151. $user = (string) Str::uuid();
  152. $thread = AgentThread::factory()->forUser($user)->create();
  153. AgentRun::factory()->for($thread, 'thread')->create([
  154. 'user_uid' => $user, 'assistant_message_id' => 'd1', 'started_at' => '2026-10-20 10:00:00', 'question' => 'Q1',
  155. 'plan' => ['lemmas' => ['mettā'], 'keywords' => [['lemma' => 'mettā', 'tags' => 'sutta']]],
  156. ]);
  157. AgentRun::factory()->for($thread, 'thread')->create(['user_uid' => $user, 'started_at' => '2026-10-20 10:00:30', 'question' => 'Q2']);
  158. AgentRun::factory()->for($thread, 'thread')->create(['user_uid' => $user, 'started_at' => '2026-10-20 10:05:00', 'question' => 'secret', 'improve' => false]);
  159. $this->postJson('/api/v3/agent-events', ['message_id' => 'd1', 'kind' => 'copy'], authHeader($user));
  160. $res = $this->getJson('/api/v3/admin/agent-runs/digest?from=2026-10-20&to=2026-10-20', feedbackAdmin())->assertOk();
  161. $items = $res->json('data');
  162. expect(array_column($items, 'question'))->toBe(['Q1', 'Q2'])
  163. ->and($items[0])->toMatchArray(['lemmas' => ['mettā'], 'keywords' => ['mettā|sutta'], 'events' => ['copy' => 1], 'next_question_after_s' => 30])
  164. ->and($items[0])->not->toHaveKeys(['user', 'user_uid', 'device_id', 'client_ip', 'account', 'answer', 'evidence'])
  165. ->and($res->json('meta.has_more'))->toBeFalse();
  166. });
  167. it('adds answer, evidence and citations to the digest on request for agent evaluation', function () {
  168. $thread = AgentThread::factory()->create();
  169. AgentRun::factory()->for($thread, 'thread')->create([
  170. 'started_at' => '2026-10-20 10:00:00', 'answer' => 'A', 'evidence' => 'E', 'citations' => [['link' => 'https://x/1-2']],
  171. ]);
  172. $admin = feedbackAdmin();
  173. $item = $this->getJson('/api/v3/admin/agent-runs/digest?from=2026-10-20&to=2026-10-20&full=1', $admin)->assertOk()->json('data.0');
  174. expect($item)->toMatchArray(['answer' => 'A', 'evidence' => 'E', 'citations' => [['link' => 'https://x/1-2']]])
  175. ->and($item)->not->toHaveKey('quality');
  176. // 带全文时条数上限收紧到 200
  177. $this->getJson('/api/v3/admin/agent-runs/digest?full=1&limit=201', $admin)->assertUnprocessable()->assertJsonValidationErrors('limit');
  178. $this->getJson('/api/v3/admin/agent-runs/digest?limit=201', $admin)->assertOk();
  179. });
  180. it('leaves soft-deleted runs out of every admin read', function () {
  181. $thread = AgentThread::factory()->create();
  182. $kept = AgentRun::factory()->for($thread, 'thread')->create(['started_at' => '2026-10-20 10:00:00', 'question' => 'kept', 'assistant_message_id' => 'm-kept']);
  183. $gone = AgentRun::factory()->for($thread, 'thread')->create(['started_at' => '2026-10-20 11:00:00', 'question' => 'gone', 'assistant_message_id' => 'm-gone']);
  184. $gone->steps()->create(['seq' => 0, 'kind' => 'tool', 'name' => 'wikipali_search']);
  185. $this->postJson('/api/v3/agent-feedback', ['message_id' => 'm-gone', 'rating' => -1, 'device_id' => (string) Str::uuid()]);
  186. $gone->steps()->delete();
  187. $gone->delete();
  188. $admin = feedbackAdmin();
  189. expect($this->getJson('/api/v3/admin/agent-runs', $admin)->json('data.*.question'))->toBe(['kept']);
  190. $this->getJson("/api/v3/admin/agent-runs/{$gone->run_id}", $admin)->assertNotFound();
  191. expect($this->getJson('/api/v3/admin/agent-runs/digest?from=2026-10-20&to=2026-10-20', $admin)->json('data.*.question'))->toBe(['kept']);
  192. $stats = $this->getJson('/api/v3/admin/agent-stats?from=2026-10-20&to=2026-10-20', $admin)->json('data');
  193. expect($stats['summary']['runs'])->toBe(1)
  194. ->and($stats['tools'])->toBe([]);
  195. // 评价本身还在,只是不再带出那次问答
  196. $this->getJson('/api/v3/admin/agent-feedback', $admin)
  197. ->assertJsonCount(1, 'data')
  198. ->assertJsonPath('data.0.run_id', null)
  199. ->assertJsonPath('data.0.question', null);
  200. });