[FEAT] Knowledge Base RAG multi-livello (SYSTEM/FIRM/ORG) + Qdrant + Voyage
- KnowledgeBaseController: ingest, list, firmOrgs, search, delete - VectorService (Qdrant + buildAuthzFilter), EmbedService (Voyage), RagService (pipeline) - AIService::askWithRag con fallback graceful - docker-compose: servizio qdrant + env Voyage (chiave da .env/vault, no hardcoded) - SQL 012 consulting_firms, 013 firm_assignments + kb_uploaded_documents - public/kb.html + kb.js (upload, lista, search preview) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
DevEnv nis2-agile
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
9b53ca3ba1
commit
a7a21faa82
@@ -564,4 +564,58 @@ PROMPT;
|
||||
'model_used' => $this->model,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Migration 012-014: Q&A grounded sulla KB multi-livello.
|
||||
*
|
||||
* Esegue una RAG search sui documenti visibili all'utente (SYSTEM/FIRM/ORG)
|
||||
* e inietta i top-K chunks nel system prompt prima di chiamare Claude.
|
||||
*
|
||||
* Se Voyage/Qdrant non sono disponibili, ricade su Claude diretto senza grounding.
|
||||
*
|
||||
* @param string $question Domanda dell'utente
|
||||
* @param array $userContext ['user_id', 'organization_id', 'consulting_firm_id']
|
||||
* @return array ['answer'=>string, 'sources'=>array, 'rag_used'=>bool]
|
||||
*/
|
||||
public function askWithRag(string $question, array $userContext): array
|
||||
{
|
||||
$sources = [];
|
||||
$contextBlock = '';
|
||||
$ragUsed = false;
|
||||
|
||||
// Tenta RAG: se fallisce, prosegui senza grounding (degradazione graceful)
|
||||
try {
|
||||
require_once __DIR__ . '/RagService.php';
|
||||
$rag = new RagService();
|
||||
$hits = $rag->searchForUser($question, $userContext, 5, 0.28);
|
||||
if (!empty($hits)) {
|
||||
$contextBlock = $rag->formatContext($hits);
|
||||
$sources = array_map(fn($h) => [
|
||||
'title' => $h['title'],
|
||||
'scope' => $h['scope'],
|
||||
'score' => $h['score'],
|
||||
], $hits);
|
||||
$ragUsed = true;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
error_log('[AIService::askWithRag] RAG failed, fallback diretto: ' . $e->getMessage());
|
||||
}
|
||||
|
||||
$systemPrompt = "Sei un esperto consulente di cybersecurity NIS2 (EU 2022/2555) e D.Lgs. 138/2024.\n"
|
||||
. "Rispondi in modo preciso e cita le fonti del contesto quando rilevanti.\n";
|
||||
if (!empty($contextBlock)) {
|
||||
$systemPrompt .= "\n## Contesto documentale (knowledge base)\n" . $contextBlock
|
||||
. "\n\nQuando rispondi, cita esplicitamente i numeri tra parentesi quadre [1], [2], ... che corrispondono ai documenti del contesto.";
|
||||
} else {
|
||||
$systemPrompt .= "\nNon e' disponibile contesto documentale specifico per questa domanda. Rispondi con la tua conoscenza generale e indica esplicitamente che non hai trovato fonti nella knowledge base.";
|
||||
}
|
||||
|
||||
$answer = $this->callAPI($question, $systemPrompt);
|
||||
|
||||
return [
|
||||
'answer' => $answer,
|
||||
'sources' => $sources,
|
||||
'rag_used' => $ragUsed,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
<?php
|
||||
/**
|
||||
* NIS2 Agile - EmbedService
|
||||
*
|
||||
* Client minimale per Voyage AI embeddings (voyage-3-lite, 1024 dim).
|
||||
*/
|
||||
|
||||
class EmbedService
|
||||
{
|
||||
public int $dims = 512;
|
||||
private string $apiKey;
|
||||
private string $model;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
// PHP-FPM Alpine non popola env via getenv() (clear_env). Multi-source lookup.
|
||||
// La chiave vive in .env (gitignored) + vault-steward; nessun segreto hardcoded.
|
||||
$this->apiKey = getenv('VOYAGE_API_KEY')
|
||||
?: ($_SERVER['VOYAGE_API_KEY'] ?? '')
|
||||
?: ($_ENV['VOYAGE_API_KEY'] ?? '')
|
||||
?: (class_exists('Env') ? Env::get('VOYAGE_API_KEY', '') : '');
|
||||
$this->model = getenv('VOYAGE_MODEL')
|
||||
?: ($_SERVER['VOYAGE_MODEL'] ?? null)
|
||||
?: ($_ENV['VOYAGE_MODEL'] ?? null)
|
||||
?: 'voyage-3-lite';
|
||||
if (empty($this->apiKey)) {
|
||||
throw new RuntimeException('VOYAGE_API_KEY non configurata');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return float[] Vettore embedding 1024-dim
|
||||
*/
|
||||
public function embed(string $text): array
|
||||
{
|
||||
$ch = curl_init('https://api.voyageai.com/v1/embeddings');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_POST => true,
|
||||
CURLOPT_HTTPHEADER => [
|
||||
'Content-Type: application/json',
|
||||
'Authorization: Bearer ' . $this->apiKey,
|
||||
],
|
||||
CURLOPT_POSTFIELDS => json_encode([
|
||||
'input' => [$text],
|
||||
'model' => $this->model,
|
||||
'input_type' => 'document',
|
||||
'output_dimension' => 512,
|
||||
]),
|
||||
CURLOPT_TIMEOUT => 30,
|
||||
]);
|
||||
$raw = curl_exec($ch);
|
||||
$status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
if ($status !== 200 || !$raw) {
|
||||
throw new RuntimeException("Voyage embed failed (HTTP $status): " . substr((string)$raw, 0, 200));
|
||||
}
|
||||
$data = json_decode($raw, true);
|
||||
$vec = $data['data'][0]['embedding'] ?? null;
|
||||
if (!is_array($vec)) {
|
||||
throw new RuntimeException('Voyage response without embedding: ' . substr($raw, 0, 200));
|
||||
}
|
||||
return $vec;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
/**
|
||||
* NIS2 Agile - RagService
|
||||
*
|
||||
* Combina EmbedService + VectorService per cercare nella KB multi-livello
|
||||
* filtrando per il contesto utente (Migration 012-014).
|
||||
*/
|
||||
|
||||
require_once __DIR__ . '/VectorService.php';
|
||||
require_once __DIR__ . '/EmbedService.php';
|
||||
|
||||
class RagService
|
||||
{
|
||||
private VectorService $vector;
|
||||
private EmbedService $embed;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->vector = new VectorService();
|
||||
$this->embed = new EmbedService();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cerca i top-k chunks visibili all'utente.
|
||||
*
|
||||
* @param array $userContext ['user_id', 'organization_id', 'consulting_firm_id']
|
||||
* @return array Lista chunks con title, content, score, scope
|
||||
*/
|
||||
public function searchForUser(string $question, array $userContext, int $topK = 5, float $minScore = 0.28): array
|
||||
{
|
||||
$vector = $this->embed->embed($question);
|
||||
$filter = VectorService::buildAuthzFilter($userContext);
|
||||
$hits = $this->vector->search($vector, $filter, $topK, $minScore);
|
||||
|
||||
$out = [];
|
||||
foreach ($hits as $h) {
|
||||
$p = $h['payload'] ?? [];
|
||||
$out[] = [
|
||||
'id' => $h['id'] ?? null,
|
||||
'score' => round($h['score'] ?? 0, 4),
|
||||
'title' => $p['title'] ?? '',
|
||||
'content' => $p['chunk'] ?? '',
|
||||
'scope' => $p['scope'] ?? null,
|
||||
'source' => $p['source'] ?? null,
|
||||
'lang' => $p['lang'] ?? 'it',
|
||||
];
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compatta i risultati in un blocco di testo da iniettare nel system prompt Claude.
|
||||
*/
|
||||
public function formatContext(array $hits): string
|
||||
{
|
||||
if (empty($hits)) return '';
|
||||
$blocks = [];
|
||||
foreach ($hits as $i => $h) {
|
||||
$idx = $i + 1;
|
||||
$blocks[] = "[$idx] {$h['title']} (scope={$h['scope']}, score={$h['score']})\n{$h['content']}";
|
||||
}
|
||||
return implode("\n\n---\n\n", $blocks);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
<?php
|
||||
/**
|
||||
* NIS2 Agile - VectorService
|
||||
*
|
||||
* Client minimale Qdrant + filtro multi-livello (Migration 012-014).
|
||||
* Modello a 3 livelli (SYSTEM/FIRM/ORG) coerente con TRPG e SustainAI.
|
||||
*/
|
||||
|
||||
class VectorService
|
||||
{
|
||||
private string $qdrantUrl;
|
||||
private string $collection;
|
||||
|
||||
public function __construct(string $collection = 'nis2_kb')
|
||||
{
|
||||
// PHP-FPM Alpine non popola correttamente env via getenv() (clear_env non
|
||||
// applicato + bug DNS musl per hostname senza dots). Workaround: leggi da
|
||||
// multiple sources e ricadi su IP statico del container Qdrant.
|
||||
$url = getenv('QDRANT_URL')
|
||||
?: ($_SERVER['QDRANT_URL'] ?? null)
|
||||
?: ($_ENV['QDRANT_URL'] ?? null)
|
||||
?: 'http://172.21.0.5:6333'; // IP nis2-qdrant nella docker_nis2-network
|
||||
$this->qdrantUrl = rtrim($url, '/');
|
||||
$this->collection = $collection;
|
||||
}
|
||||
|
||||
public function ensureCollection(int $dims = 1024): void
|
||||
{
|
||||
$info = $this->request('GET', "/collections/{$this->collection}");
|
||||
if ($info['status'] === 200) return;
|
||||
|
||||
$this->request('PUT', "/collections/{$this->collection}", [
|
||||
'vectors' => ['size' => $dims, 'distance' => 'Cosine'],
|
||||
]);
|
||||
}
|
||||
|
||||
public function upsertBatch(array $points): void
|
||||
{
|
||||
if (empty($points)) return;
|
||||
$resp = $this->request('PUT', "/collections/{$this->collection}/points?wait=true", [
|
||||
'points' => $points,
|
||||
]);
|
||||
if ($resp['status'] !== 200) {
|
||||
throw new RuntimeException('Qdrant upsert failed (HTTP ' . $resp['status'] . '): ' . json_encode($resp['body']));
|
||||
}
|
||||
}
|
||||
|
||||
public function deleteByFilter(array $filter): void
|
||||
{
|
||||
$this->request('POST', "/collections/{$this->collection}/points/delete", [
|
||||
'filter' => $filter,
|
||||
]);
|
||||
}
|
||||
|
||||
public function setPayloadByFilter(array $payload, array $filter): void
|
||||
{
|
||||
$this->request('POST', "/collections/{$this->collection}/points/payload", [
|
||||
'payload' => $payload,
|
||||
'filter' => $filter,
|
||||
]);
|
||||
}
|
||||
|
||||
public function search(array $vector, array $filter = [], int $limit = 8, float $minScore = 0.28): array
|
||||
{
|
||||
$body = [
|
||||
'vector' => $vector,
|
||||
'limit' => $limit,
|
||||
'with_payload' => true,
|
||||
'score_threshold'=> $minScore,
|
||||
];
|
||||
if (!empty($filter)) {
|
||||
$body['filter'] = $filter;
|
||||
}
|
||||
$resp = $this->request('POST', "/collections/{$this->collection}/points/search", $body);
|
||||
if ($resp['status'] !== 200) {
|
||||
return [];
|
||||
}
|
||||
return $resp['body']['result'] ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Filtro a 3 livelli (SYSTEM/FIRM/ORG) basato sull'utente.
|
||||
* Restituisce SOLO chunks visibili a quell'utente.
|
||||
*
|
||||
* @param array $userContext ['user_id'=>int, 'organization_id'=>int|null, 'consulting_firm_id'=>int|null]
|
||||
*/
|
||||
public static function buildAuthzFilter(array $userContext): array
|
||||
{
|
||||
$firmId = isset($userContext['consulting_firm_id']) && $userContext['consulting_firm_id'] !== null
|
||||
? (int)$userContext['consulting_firm_id'] : null;
|
||||
$orgId = isset($userContext['organization_id']) && $userContext['organization_id'] !== null
|
||||
? (int)$userContext['organization_id'] : null;
|
||||
|
||||
$should = [];
|
||||
|
||||
// L0 SYSTEM: vendor knowledge (sempre visibile)
|
||||
$should[] = ['key' => 'scope', 'match' => ['value' => 'SYSTEM']];
|
||||
|
||||
// L1 FIRM: KB del proprio studio (visibile a tutti i collaboratori)
|
||||
if ($firmId !== null) {
|
||||
$should[] = [
|
||||
'must' => [
|
||||
['key' => 'scope', 'match' => ['value' => 'FIRM']],
|
||||
['key' => 'consulting_firm_id', 'match' => ['value' => $firmId]],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
if ($orgId !== null) {
|
||||
// L1 FIRM con sharing esplicito alla organization corrente
|
||||
$should[] = [
|
||||
'must' => [
|
||||
['key' => 'scope', 'match' => ['value' => 'FIRM']],
|
||||
['key' => 'shared_with_orgs', 'match' => ['any' => [$orgId]]],
|
||||
],
|
||||
];
|
||||
// L2 ORG: chunk dell'organizzazione corrente
|
||||
$should[] = [
|
||||
'must' => [
|
||||
['key' => 'scope', 'match' => ['value' => 'ORG']],
|
||||
['key' => 'organization_id', 'match' => ['value' => $orgId]],
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
return ['should' => $should];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{status:int, body:array}
|
||||
*/
|
||||
private function request(string $method, string $path, ?array $body = null): array
|
||||
{
|
||||
$url = $this->qdrantUrl . $path;
|
||||
$ch = curl_init($url);
|
||||
|
||||
$opts = [
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
CURLOPT_CUSTOMREQUEST => $method,
|
||||
CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
|
||||
CURLOPT_CONNECTTIMEOUT => 3,
|
||||
CURLOPT_TIMEOUT => 30,
|
||||
];
|
||||
if ($body !== null) {
|
||||
$opts[CURLOPT_POSTFIELDS] = json_encode($body);
|
||||
}
|
||||
curl_setopt_array($ch, $opts);
|
||||
|
||||
$raw = curl_exec($ch);
|
||||
$status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
curl_close($ch);
|
||||
|
||||
return [
|
||||
'status' => $status,
|
||||
'body' => $raw ? (json_decode($raw, true) ?? []) : [],
|
||||
];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user