Évaluer la pertinence des réponses des agents IA avec OpenAI et la similarité cosinus

• •Peut nécessiter les informations d'identification d'authentification de l'API cible
• •Clé API OpenAI

{
  "meta": {
    "instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9",
    "templateCredsSetupCompleted": true
  },
  "nodes": [
    {
      "id": "49b274c7-1ff2-43ba-acb9-349a8665b513",
      "name": "OpenAI Chat Model1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        148,
        180
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "77f0c130-085f-4fa5-b74e-66fd084949b7",
      "name": "Lors de la récupération d'une ligne de jeu de données",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -380,
        -140
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
      "name": "Remapper l'entrée",
      "type": "n8n-nodes-base.set",
      "position": [
        -160,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "00924b90-278f-49f5-80f2-c297df0fcc97",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.input }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
      "name": "Évaluation",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        436,
        -40
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
      "name": "Définir les champs d'entrée",
      "type": "n8n-nodes-base.set",
      "position": [
        656,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "d58952c1-d346-4fbf-881e-d5c04b6781a5",
              "name": "question",
              "type": "string",
              "value": "={{ $('When fetching a dataset row').first().json.input }}"
            },
            {
              "id": "0f10a3d0-cf6e-4715-9ded-2cee54aa62ec",
              "name": "answer",
              "type": "string",
              "value": "={{ $json.output }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
      "name": "Aucune opération, ne rien faire",
      "type": "n8n-nodes-base.noOp",
      "position": [
        656,
        60
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
      "name": "Agent IA",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        60,
        -40
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 2
    },
    {
      "id": "54596db1-4ba7-414c-88cf-250f528d6c21",
      "name": "Lors de la réception d'un message de chat",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -160,
        60
      ],
      "webhookId": "ba1fadeb-b566-469a-97b3-3159a99f1805",
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "4184f202-48f5-4e58-a0e8-0227a23553f3",
      "name": "Mettre à jour la sortie",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2132,
        -140
      ],
      "parameters": {
        "outputs": {
          "values": [
            {
              "outputName": "output",
              "outputValue": "={{ $('Set Input Fields').first().json.answer }}"
            },
            {
              "outputName": "score",
              "outputValue": "={{ $json.score }}"
            }
          ]
        },
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "fe7b41d2-e382-4f59-8b0c-06361062581d",
      "name": "Mettre à jour les métriques",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2352,
        -140
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "1fd7759c-f4ef-4eda-87ad-9d9563b63e99",
              "name": "score",
              "type": "number",
              "value": "={{ $json.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "04f714cd-99ce-45a9-bc43-351fb6ea14d4",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        920,
        80
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "d821b0ec-93c9-46e1-ab03-280aeb9b87b1",
      "name": "Analyseur de sortie structurée",
      "type": "@n8n/n8n-nodes-langchain.outputParserStructured",
      "position": [
        1040,
        80
      ],
      "parameters": {
        "jsonSchemaExample": "{\n  \"question\": \"Where was Albert Einstein born?\",\n  \"noncommittal\": 0\n}"
      },
      "typeVersion": 1.2
    },
    {
      "id": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
      "name": "Pertinence de la réponse",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        900,
        -140
      ],
      "parameters": {
        "text": "={{ $json.answer }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=Generate a question for the given answer and Identify if answer is noncommittal. Give noncommittal as 1 if the answer is noncommittal and 0 if the answer is committal. A noncommittal answer is one that is evasive, vague, or ambiguous. For example, \"I don't know\" or \"I'm not sure\" are noncommittal answers\""
            }
          ]
        },
        "promptType": "define",
        "hasOutputParser": true
      },
      "typeVersion": 1.7
    },
    {
      "id": "8e3df14a-7754-4257-8209-a24cfa30dbbc",
      "name": "Note adhésive1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -452,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nThe Evaluations Trigger is a separate execution which does not affect your production workflow in any way. It is manually triggered and automatically pulled datasets from the assigned Google Sheet."
      },
      "typeVersion": 1
    },
    {
      "id": "eeeac9d0-39c6-4d17-8d93-9c60b6c612f8",
      "name": "Note adhésive",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        820,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 1200,
        "height": 720,
        "content": "## 2. Answer Relevance: How relevant is the agent response to the user question?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nFor this evaluation, we analyse the agent's answer and ask another AI to generate a question for it - kinda like the game show [Jeopardy!](https://en.wikipedia.org/wiki/Jeopardy!).\nWe do this in order to compare the generated question with the original question for the test. If the questions are similar, then relevancy is high."
      },
      "typeVersion": 1
    },
    {
      "id": "c86f66e2-9997-4ca6-9ec6-cfa185d404f2",
      "name": "Note adhésive3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -920,
        -340
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Relevance\" which in this scenario, measures the relevance of the agent's response to the user's question.\n\nThe scoring approach is adapted from [https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py](https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py)\n\n### How it works\n* This evaluation works best for Q&A agents.\n* For our scoring, we analyse the agent's response and ask another AI to generate a question from it. This generated question is then compared to the original question using cosine similarity.\n* A high score indicates relevance and the agent's successful ability to answer the question whereas a low score means agent may have added too much irrelevant info, went off script or hallucinated.\n\n### Requirements\n* n8n version 1.94+\n* Check out this Google Sheet for a sample data [https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing](https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing)\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
      },
      "typeVersion": 1
    },
    {
      "id": "3bdd3254-7cdf-455a-ae63-9639069993bc",
      "name": "Obtenir les embeddings",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1460,
        -140
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.data }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
      "name": "Calculer le score de similarité",
      "type": "n8n-nodes-base.code",
      "position": [
        1660,
        -140
      ],
      "parameters": {
        "jsCode": "const [vectorsA, vectorsB] = $input.all().map(item => item.json.data[0].embedding);\nconst score = cosineSimilarity(vectorsA, vectorsB);\n\nreturn { json: { similarityScore: score } };\n\nfunction cosineSimilarity(a, b) {  \n  let dotProduct = normA = normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dotProduct += a[i] * b[i];\n    normA += a[i] ** 2;\n    normB += b[i] ** 2;\n  }\n  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n}"
      },
      "typeVersion": 2
    },
    {
      "id": "69c30814-9725-426d-9572-9f92bd81dea5",
      "name": "Questions vers éléments",
      "type": "n8n-nodes-base.code",
      "position": [
        1260,
        -140
      ],
      "parameters": {
        "jsCode": "return [\n  { json: { data: $('Set Input Fields').first().json.question } },\n  { json: { data: $input.first().json.output.question } }\n]"
      },
      "typeVersion": 2
    },
    {
      "id": "83be23ff-6857-4265-b1d0-f4b682c66903",
      "name": "Calculer le score de pertinence",
      "type": "n8n-nodes-base.set",
      "position": [
        1860,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "c5abdf27-cb9e-4969-b669-25c6fbd135f1",
              "name": "score",
              "type": "number",
              "value": "={{ $json.similarityScore * Math.abs(!$('Answer Relevance').first().json.output.noncommittal) }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    }
  ],
  "pinData": {},
  "connections": {
    "1aea189f-3e7d-45d2-9802-d44b1dce4797": {
      "main": [
        [
          {
            "node": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76": {
      "main": [
        [
          {
            "node": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "4184f202-48f5-4e58-a0e8-0227a23553f3": {
      "main": [
        [
          {
            "node": "fe7b41d2-e382-4f59-8b0c-06361062581d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "3bdd3254-7cdf-455a-ae63-9639069993bc": {
      "main": [
        [
          {
            "node": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "be975ef4-c50b-4bf9-b871-cc8d479661eb": {
      "main": [
        [
          {
            "node": "69c30814-9725-426d-9572-9f92bd81dea5",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6": {
      "main": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "04f714cd-99ce-45a9-bc43-351fb6ea14d4": {
      "ai_languageModel": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "49b274c7-1ff2-43ba-acb9-349a8665b513": {
      "ai_languageModel": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "69c30814-9725-426d-9572-9f92bd81dea5": {
      "main": [
        [
          {
            "node": "3bdd3254-7cdf-455a-ae63-9639069993bc",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d821b0ec-93c9-46e1-ab03-280aeb9b87b1": {
      "ai_outputParser": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_outputParser",
            "index": 0
          }
        ]
      ]
    },
    "83be23ff-6857-4265-b1d0-f4b682c66903": {
      "main": [
        [
          {
            "node": "4184f202-48f5-4e58-a0e8-0227a23553f3",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d": {
      "main": [
        [
          {
            "node": "83be23ff-6857-4265-b1d0-f4b682c66903",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "54596db1-4ba7-414c-88cf-250f528d6c21": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "77f0c130-085f-4fa5-b74e-66fd084949b7": {
      "main": [
        [
          {
            "node": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}