OpenAI와 코사인 유사도를 사용하여 AI 대리자 응답 관련성 평가

• •대상 API의 인증 정보가 필요할 수 있음
• •OpenAI API Key

{
  "meta": {
    "instanceId": "408f9fb9940c3cb18ffdef0e0150fe342d6e655c3a9fac21f0f644e8bedabcd9",
    "templateCredsSetupCompleted": true
  },
  "nodes": [
    {
      "id": "49b274c7-1ff2-43ba-acb9-349a8665b513",
      "name": "OpenAI 채팅 모델1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        148,
        180
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "77f0c130-085f-4fa5-b74e-66fd084949b7",
      "name": "데이터셋 행 가져오기 시",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -380,
        -140
      ],
      "parameters": {
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
      "name": "입력 재매핑",
      "type": "n8n-nodes-base.set",
      "position": [
        -160,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "00924b90-278f-49f5-80f2-c297df0fcc97",
              "name": "chatInput",
              "type": "string",
              "value": "={{ $json.input }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
      "name": "평가",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        436,
        -40
      ],
      "parameters": {
        "operation": "checkIfEvaluating"
      },
      "typeVersion": 4.6
    },
    {
      "id": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
      "name": "입력 필드 설정",
      "type": "n8n-nodes-base.set",
      "position": [
        656,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "d58952c1-d346-4fbf-881e-d5c04b6781a5",
              "name": "question",
              "type": "string",
              "value": "={{ $('When fetching a dataset row').first().json.input }}"
            },
            {
              "id": "0f10a3d0-cf6e-4715-9ded-2cee54aa62ec",
              "name": "answer",
              "type": "string",
              "value": "={{ $json.output }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    },
    {
      "id": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
      "name": "작업 없음, 아무 동작도 수행하지 않음",
      "type": "n8n-nodes-base.noOp",
      "position": [
        656,
        60
      ],
      "parameters": {},
      "typeVersion": 1
    },
    {
      "id": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
      "name": "AI 에이전트",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        60,
        -40
      ],
      "parameters": {
        "options": {}
      },
      "typeVersion": 2
    },
    {
      "id": "54596db1-4ba7-414c-88cf-250f528d6c21",
      "name": "채팅 메시지 수신 시",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -160,
        60
      ],
      "webhookId": "ba1fadeb-b566-469a-97b3-3159a99f1805",
      "parameters": {
        "options": {}
      },
      "typeVersion": 1.1
    },
    {
      "id": "4184f202-48f5-4e58-a0e8-0227a23553f3",
      "name": "출력 업데이트",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2132,
        -140
      ],
      "parameters": {
        "outputs": {
          "values": [
            {
              "outputName": "output",
              "outputValue": "={{ $('Set Input Fields').first().json.answer }}"
            },
            {
              "outputName": "score",
              "outputValue": "={{ $json.score }}"
            }
          ]
        },
        "sheetName": {
          "__rl": true,
          "mode": "list",
          "value": 2015506025,
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit#gid=2015506025",
          "cachedResultName": "Relevance"
        },
        "documentId": {
          "__rl": true,
          "mode": "list",
          "value": "1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y",
          "cachedResultUrl": "https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=drivesdk",
          "cachedResultName": "96. Evaluations Test"
        }
      },
      "credentials": {
        "googleSheetsOAuth2Api": {
          "id": "XHvC7jIRR8A2TlUl",
          "name": "Google Sheets account"
        }
      },
      "typeVersion": 4.6
    },
    {
      "id": "fe7b41d2-e382-4f59-8b0c-06361062581d",
      "name": "메트릭 업데이트",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2352,
        -140
      ],
      "parameters": {
        "metrics": {
          "assignments": [
            {
              "id": "1fd7759c-f4ef-4eda-87ad-9d9563b63e99",
              "name": "score",
              "type": "number",
              "value": "={{ $json.score }}"
            }
          ]
        },
        "operation": "setMetrics"
      },
      "typeVersion": 4.6
    },
    {
      "id": "04f714cd-99ce-45a9-bc43-351fb6ea14d4",
      "name": "OpenAI 채팅 모델",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        920,
        80
      ],
      "parameters": {
        "model": {
          "__rl": true,
          "mode": "list",
          "value": "gpt-4.1-mini",
          "cachedResultName": "gpt-4.1-mini"
        },
        "options": {}
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 1.2
    },
    {
      "id": "d821b0ec-93c9-46e1-ab03-280aeb9b87b1",
      "name": "구조화된 출력 파서",
      "type": "@n8n/n8n-nodes-langchain.outputParserStructured",
      "position": [
        1040,
        80
      ],
      "parameters": {
        "jsonSchemaExample": "{\n  \"question\": \"Where was Albert Einstein born?\",\n  \"noncommittal\": 0\n}"
      },
      "typeVersion": 1.2
    },
    {
      "id": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
      "name": "응답 관련성",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        900,
        -140
      ],
      "parameters": {
        "text": "={{ $json.answer }}",
        "batching": {},
        "messages": {
          "messageValues": [
            {
              "message": "=Generate a question for the given answer and Identify if answer is noncommittal. Give noncommittal as 1 if the answer is noncommittal and 0 if the answer is committal. A noncommittal answer is one that is evasive, vague, or ambiguous. For example, \"I don't know\" or \"I'm not sure\" are noncommittal answers\""
            }
          ]
        },
        "promptType": "define",
        "hasOutputParser": true
      },
      "typeVersion": 1.7
    },
    {
      "id": "8e3df14a-7754-4257-8209-a24cfa30dbbc",
      "name": "스티커 노트1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -452,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nThe Evaluations Trigger is a separate execution which does not affect your production workflow in any way. It is manually triggered and automatically pulled datasets from the assigned Google Sheet."
      },
      "typeVersion": 1
    },
    {
      "id": "eeeac9d0-39c6-4d17-8d93-9c60b6c612f8",
      "name": "스티커 노트",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        820,
        -400
      ],
      "parameters": {
        "color": 7,
        "width": 1200,
        "height": 720,
        "content": "## 2. Answer Relevance: How relevant is the agent response to the user question?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.evaluationTrigger)\n\nFor this evaluation, we analyse the agent's answer and ask another AI to generate a question for it - kinda like the game show [Jeopardy!](https://en.wikipedia.org/wiki/Jeopardy!).\nWe do this in order to compare the generated question with the original question for the test. If the questions are similar, then relevancy is high."
      },
      "typeVersion": 1
    },
    {
      "id": "c86f66e2-9997-4ca6-9ec6-cfa185d404f2",
      "name": "스티커 노트3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -920,
        -340
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Relevance\" which in this scenario, measures the relevance of the agent's response to the user's question.\n\nThe scoring approach is adapted from [https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py](https://github.com/explodinggradients/ragas/blob/main/ragas/src/ragas/metrics/_answer_relevance.py)\n\n### How it works\n* This evaluation works best for Q&A agents.\n* For our scoring, we analyse the agent's response and ask another AI to generate a question from it. This generated question is then compared to the original question using cosine similarity.\n* A high score indicates relevance and the agent's successful ability to answer the question whereas a low score means agent may have added too much irrelevant info, went off script or hallucinated.\n\n### Requirements\n* n8n version 1.94+\n* Check out this Google Sheet for a sample data [https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing](https://docs.google.com/spreadsheets/d/1YOnu2JJjlxd787AuYcg-wKbkjyjyZFgASYVV0jsij5Y/edit?usp=sharing)\n\n\n### Need Help?\nJoin the [Discord](https://discord.com/invite/XPKeKXeB7d) or ask in the [Forum](https://community.n8n.io/)!\n\nHappy Hacking!"
      },
      "typeVersion": 1
    },
    {
      "id": "3bdd3254-7cdf-455a-ae63-9639069993bc",
      "name": "임베딩 가져오기",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1460,
        -140
      ],
      "parameters": {
        "url": "https://api.openai.com/v1/embeddings",
        "method": "POST",
        "options": {},
        "sendBody": true,
        "authentication": "predefinedCredentialType",
        "bodyParameters": {
          "parameters": [
            {
              "name": "input",
              "value": "={{ $json.data }}"
            },
            {
              "name": "model",
              "value": "text-embedding-3-small"
            },
            {
              "name": "encoding_format",
              "value": "float"
            }
          ]
        },
        "nodeCredentialType": "openAiApi"
      },
      "credentials": {
        "openAiApi": {
          "id": "8gccIjcuf3gvaoEr",
          "name": "OpenAi account"
        }
      },
      "typeVersion": 4.2
    },
    {
      "id": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
      "name": "유사도 점수 계산",
      "type": "n8n-nodes-base.code",
      "position": [
        1660,
        -140
      ],
      "parameters": {
        "jsCode": "const [vectorsA, vectorsB] = $input.all().map(item => item.json.data[0].embedding);\nconst score = cosineSimilarity(vectorsA, vectorsB);\n\nreturn { json: { similarityScore: score } };\n\nfunction cosineSimilarity(a, b) {  \n  let dotProduct = normA = normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    dotProduct += a[i] * b[i];\n    normA += a[i] ** 2;\n    normB += b[i] ** 2;\n  }\n  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));\n}"
      },
      "typeVersion": 2
    },
    {
      "id": "69c30814-9725-426d-9572-9f92bd81dea5",
      "name": "질문을 항목으로 변환",
      "type": "n8n-nodes-base.code",
      "position": [
        1260,
        -140
      ],
      "parameters": {
        "jsCode": "return [\n  { json: { data: $('Set Input Fields').first().json.question } },\n  { json: { data: $input.first().json.output.question } }\n]"
      },
      "typeVersion": 2
    },
    {
      "id": "83be23ff-6857-4265-b1d0-f4b682c66903",
      "name": "관련성 점수 계산",
      "type": "n8n-nodes-base.set",
      "position": [
        1860,
        -140
      ],
      "parameters": {
        "options": {},
        "assignments": {
          "assignments": [
            {
              "id": "c5abdf27-cb9e-4969-b669-25c6fbd135f1",
              "name": "score",
              "type": "number",
              "value": "={{ $json.similarityScore * Math.abs(!$('Answer Relevance').first().json.output.noncommittal) }}"
            }
          ]
        }
      },
      "typeVersion": 3.4
    }
  ],
  "pinData": {},
  "connections": {
    "1aea189f-3e7d-45d2-9802-d44b1dce4797": {
      "main": [
        [
          {
            "node": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76": {
      "main": [
        [
          {
            "node": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "4184f202-48f5-4e58-a0e8-0227a23553f3": {
      "main": [
        [
          {
            "node": "fe7b41d2-e382-4f59-8b0c-06361062581d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "3bdd3254-7cdf-455a-ae63-9639069993bc": {
      "main": [
        [
          {
            "node": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "be975ef4-c50b-4bf9-b871-cc8d479661eb": {
      "main": [
        [
          {
            "node": "69c30814-9725-426d-9572-9f92bd81dea5",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6": {
      "main": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "04f714cd-99ce-45a9-bc43-351fb6ea14d4": {
      "ai_languageModel": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "49b274c7-1ff2-43ba-acb9-349a8665b513": {
      "ai_languageModel": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "69c30814-9725-426d-9572-9f92bd81dea5": {
      "main": [
        [
          {
            "node": "3bdd3254-7cdf-455a-ae63-9639069993bc",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "d821b0ec-93c9-46e1-ab03-280aeb9b87b1": {
      "ai_outputParser": [
        [
          {
            "node": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
            "type": "ai_outputParser",
            "index": 0
          }
        ]
      ]
    },
    "83be23ff-6857-4265-b1d0-f4b682c66903": {
      "main": [
        [
          {
            "node": "4184f202-48f5-4e58-a0e8-0227a23553f3",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d": {
      "main": [
        [
          {
            "node": "83be23ff-6857-4265-b1d0-f4b682c66903",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "54596db1-4ba7-414c-88cf-250f528d6c21": {
      "main": [
        [
          {
            "node": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "77f0c130-085f-4fa5-b74e-66fd084949b7": {
      "main": [
        [
          {
            "node": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}