diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c455c58..2d84450 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ -# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @opensearch-project/search-request-processor-team \ No newline at end of file +# This should match the MAINTAINERS.md file in the root of this repo +* @msfroh @macohen @noCharger @mingshl @sejli \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 19cd65f..7a52fcd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,4 +12,4 @@ - [ ] Commits are signed as per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. -For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/k-NN/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). +For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/search-processor/blob/main/CONTRIBUTING.md). diff --git a/.github/draft-release-notes-config.yml b/.github/draft-release-notes-config.yml index d168727..e27eaaa 100644 --- a/.github/draft-release-notes-config.yml +++ b/.github/draft-release-notes-config.yml @@ -16,25 +16,28 @@ replacers: categories: - title: 'Breaking Changes' labels: - - 'Breaking Changes' + - 'breaking' + - title: 'Security' + labels: + - 'security fix' - title: 'Features' labels: - - 'Features' + - 'feature' - title: 'Enhancements' labels: - - 'Enhancements' + - 'enhancement' - title: 'Bug Fixes' labels: - - 'Bug Fixes' + - 'bug' - title: 'Infrastructure' labels: - - 'Infrastructure' + - 'infrastructure' - title: 'Documentation' labels: - - 'Documentation' + - 'documentation' - title: 'Maintenance' labels: - - 'Maintenance' + - 'maintenance' - title: 'Refactoring' labels: - - 'Refactoring' \ No newline at end of file + - 'refactoring' \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 61bfc57..6a31d4d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,7 +15,7 @@ jobs: Build-search-request-processor: strategy: matrix: - java: [11, 17] + java: [21] os: [ubuntu-latest, macos-latest, windows-latest] name: Build and Test Search Request Processor Plugin @@ -23,10 +23,10 @@ jobs: steps: - name: Checkout Search Request Processor - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} @@ -42,7 +42,7 @@ jobs: - name: Upload Coverage Report if: ${{matrix.os}} == 'ubuntu' - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@29386c70ef20e286228c72b668a06fd0e8399192 # v1 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml new file mode 100644 index 0000000..846441e --- /dev/null +++ b/.github/workflows/add-untriaged.yml @@ -0,0 +1,22 @@ +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +permissions: + issues: write + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index e47d8d8..2609a3c 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,14 +15,14 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Backport - uses: VachaShah/backport@v1.1.4 + uses: VachaShah/backport@28c49d91ceec57d7c9f625f1031c1a4d637251f5 # v1.1.4 with: github_token: ${{ steps.github_app_token.outputs.token }} branch_name: backport/backport-${{ github.event.number }} diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 750c8b5..8644c7c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -13,8 +13,8 @@ jobs: Restart-Upgrade-BWCTests-k-NN: strategy: matrix: - java: [ 11, 17 ] - bwc_version : [ "2.4.0" ] + java: [ 21 ] + bwc_version : [ "2.7.0" ] opensearch_version : [ "3.0.0-SNAPSHOT" ] name: SRP Restart-Upgrade BWC Tests @@ -24,10 +24,10 @@ jobs: steps: - name: Checkout SRP - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} @@ -41,8 +41,8 @@ jobs: Rolling-Upgrade-BWCTests-SRP: strategy: matrix: - java: [ 11, 17 ] - bwc_version: [ "2.4.0" ] + java: [ 21 ] + bwc_version: [ "2.11.1" ] opensearch_version: [ "3.0.0-SNAPSHOT" ] name: SRP Rolling-Upgrade BWC Tests @@ -52,10 +52,10 @@ jobs: steps: - name: Checkout SRP - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml index 1ab6e8b..3938005 100644 --- a/.github/workflows/create-documentation-issue.yml +++ b/.github/workflows/create-documentation-issue.yml @@ -14,22 +14,22 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: Edit the issue template run: | - echo "https://github.com/opensearch-project/k-NN/pull/${{ env.PR_NUMBER }}." >> ./.github/ISSUE_TEMPLATE/documentation-issue.md + echo "https://github.com/opensearch-project/search-processor/pull/${{ env.PR_NUMBER }}." >> ./.github/ISSUE_TEMPLATE/documentation-issue.md - name: Create Issue From File id: create-issue - uses: peter-evans/create-issue-from-file@v4 + uses: peter-evans/create-issue-from-file@433e51abf769039ee20ba1293a088ca19d573b7f # v4 with: title: Add documentation related to new feature content-filepath: ./.github/ISSUE_TEMPLATE/documentation-issue.md diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index cf30ea8..0000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index a97f9cd..156d749 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -10,6 +10,6 @@ jobs: if: startsWith(github.event.pull_request.head.ref,'backport-') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main + uses: SvanBoxel/delete-merged-branch@2b5b058e3db41a3328fd9a6a58fd4c2545a14353 # main env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml index bdafb18..7d13551 100644 --- a/.github/workflows/dependabot_pr.yml +++ b/.github/workflows/dependabot_pr.yml @@ -11,14 +11,14 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 with: token: ${{ steps.github_app_token.outputs.token }} @@ -27,7 +27,7 @@ jobs: ./gradlew updateSHAs - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4.7.2 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.7.2 with: commit_message: Updating SHAs branch: ${{ github.head_ref }} @@ -36,7 +36,7 @@ jobs: commit_options: '--signoff' - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4.7.2 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.7.2 with: commit_message: Spotless formatting branch: ${{ github.head_ref }} @@ -45,12 +45,12 @@ jobs: commit_options: '--signoff' - name: Update the changelog - uses: dangoslen/dependabot-changelog-helper@v1 + uses: dangoslen/dependabot-changelog-helper@780f7c82213ff956b1bd8cb484ba67d1fbe8b4ba # v1 with: version: 'Unreleased' - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4 with: commit_message: "Update changelog" branch: ${{ github.head_ref }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 6b3d89c..48e2bf9 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update draft release notes - uses: release-drafter/release-drafter@v5 + uses: release-drafter/release-drafter@09c613e259eb8d4e7c81c2cb00618eb5fc4575a7 # v5 with: config-name: draft-release-notes-config.yml name: Version (set here) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 3d0b81a..88f0d32 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: lychee Link Checker id: lychee - uses: lycheeverse/lychee-action@master + uses: lycheeverse/lychee-action@6da1d14f3a43098a294b7696d93d938aa8d20fc0 # master with: args: --accept=200,403,429 **/*.html **/*.md **/*.txt **/*.json env: diff --git a/.github/workflows/pr_stats.yml b/.github/workflows/pr_stats.yml new file mode 100644 index 0000000..5cb0564 --- /dev/null +++ b/.github/workflows/pr_stats.yml @@ -0,0 +1,15 @@ +name: Pull Request Stats + +on: + pull_request: + types: [opened] + +jobs: + stats: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Run pull request stats + uses: flowwer-dev/pull-request-stats@0dde6edf8b7db75684533021212c6e85fb987dbb # master diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b101f73..4d96f68 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1,6 +1,6 @@ - [Developer Guide](#developer-guide) - [Getting Started](#getting-started) - - [Fork OpenSearch search-relevance Repo](#fork-opensearch-search-processor-repo) + - [Fork OpenSearch search-processor Repo](#fork-opensearch-search-processor-repo) - [Install Prerequisites](#install-prerequisites) - [JDK 11](#jdk-11) - [Environment](#Environment) @@ -23,11 +23,11 @@ So you want to contribute code to OpenSearch Search Processor? Excellent! We're ### Fork OpenSearch Search Processor Repo -Fork [opensearch-project/OpenSearch Search Processor](https://github.com/opensearch-project/search-relevance) and clone locally. +Fork [opensearch-project/OpenSearch Search Processor](https://github.com/opensearch-project/search-processor) and clone locally. Example: ``` -git clone https://github.com/[your username]/search-relevance.git +git clone https://github.com/[your username]/search-processor.git ``` ### Install Prerequisites diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 157fbf6..12c0cd1 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,146 +1,13 @@ -- [Overview](#overview) -- [Current Maintainers](#current-maintainers) -- [Maintainer Responsibilities](#maintainer-responsibilities) - - [Uphold Code of Conduct](#uphold-code-of-conduct) - - [Prioritize Security](#prioritize-security) - - [Review Pull Requests](#review-pull-requests) - - [Triage Open Issues](#triage-open-issues) - - [Be Responsive](#be-responsive) - - [Maintain Overall Health of the Repo](#maintain-overall-health-of-the-repo) - - [Manage Roadmap](#manage-roadmap) - - [Add Continuous Integration Checks](#add-continuous-integration-checks) - - [Developer Certificate of Origin Workflow](#developer-certificate-of-origin-workflow) - - [Use Semver](#use-semver) - - [Release Frequently](#release-frequently) - - [Promote Other Maintainers](#promote-other-maintainers) - - [Describe the Repo](#describe-the-repo) -- [Becoming a Maintainer](#becoming-a-maintainer) - - [Nomination](#nomination) - - [Interest](#interest) - - [Addition](#addition) -- [Removing a Maintainer](#removing-a-maintainer) - - [Moving On](#moving-on) - - [Inactivity](#inactivity) - - [Negative Impact on the Project](#negative-impact-on-the-project) - ## Overview -This document explains who the maintainers are (see below), what they do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md). +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). ## Current Maintainers -| Maintainer | GitHub ID | Affiliation | -|-------------------------|-----------------------------------------| ----------- | -| Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | -| Michael Froh | [msfroh](https://github.com/msfroh) | Amazon | -| Mark Cohen | [macohen](https://github.com/macohen) | Amazon | -| Lior Perry | [yang-db](https://github.com/yang-db) | Amazon | -| Shenoy Pratik | [ps48](https://github.com/ps48) | Amazon | - -## Maintainer Responsibilities - -Maintainers are active and visible members of the community, and have [maintain-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and evolve code as follows. - -### Uphold Code of Conduct - -Model the behavior set forward by the [Code of Conduct](CODE_OF_CONDUCT.md) and raise any violations to other maintainers and admins. - -### Prioritize Security - -Security is your number one priority. Maintainer's Github keys must be password protected securely and any reported security vulnerabilities are addressed before features or bugs. - -Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details. - -### Review Pull Requests - -Review pull requests regularly, comment, suggest, reject, merge and close. Accept only high quality pull-requests. Provide code reviews and guidance on incoming pull requests. Don't let PRs be stale and do your best to be helpful to contributors. - -### Triage Open Issues - -Manage labels, review issues regularly, and triage by labelling them. - -All repositories in this organization have a standard set of labels, including `bug`, `documentation`, `duplicate`, `enhancement`, `good first issue`, `help wanted`, `blocker`, `invalid`, `question`, `wontfix`, and `untriaged`, along with release labels, such as `v1.0.0`, `v1.1.0`, `v2.0.0`, `patch`, and `backport`. - -Use labels to target an issue or a PR for a given release, add `help wanted` to good issues for new community members, and `blocker` for issues that scare you or need immediate attention. Request for more information from a submitter if an issue is not clear. Create new labels as needed by the project. - -### Be Responsive - -Respond to enhancement requests, and forum posts. Allocate time to reviewing and commenting on issues and conversations as they come in. - -### Maintain Overall Health of the Repo - -Keep the `main` branch at production quality at all times. Backport features as needed. Cut release branches and tags to enable future patches. - -### Manage Roadmap - -Ensure the repo highlights features that should be elevated to the project roadmap. Be clear about the feature’s status, priority, target version, and whether or not it should be elevated to the roadmap. Any feature that you want highlighted on the OpenSearch Roadmap should be tagged with "roadmap". The OpenSearch [project-meta maintainers](https://github.com/opensearch-project/project-meta/blob/main/MAINTAINERS.md) will highlight features tagged "roadmap" on the project wide [OpenSearch Roadmap](https://github.com/orgs/opensearch-project/projects/1). - -### Add Continuous Integration Checks - -Add integration checks that validate pull requests and pushes to ease the burden on Pull Request reviewers. - -#### Developer Certificate of Origin Workflow - -Validates pull requests commits are all signed with DCO, [dco.yml](./workflow/dco.yml). Example [pull request](https://github.com/opensearch-project/opensearch-ci/pull/16). - -### Use Semver - -Use and enforce [semantic versioning](https://semver.org/) and do not let breaking changes be made outside of major releases. - -### Release Frequently - -Make frequent project releases to the community. - -### Promote Other Maintainers - -Assist, add, and remove [MAINTAINERS](MAINTAINERS.md). Exercise good judgement, and propose high quality contributors to become co-maintainers. See [Becoming a Maintainer](#becoming-a-maintainer) for more information. - -### Describe the Repo - -Make sure the repo has a well-written, accurate, and complete description. See [opensearch-project/.github#38](https://github.com/opensearch-project/.github/issues/38) for some helpful tips to describe your repo. - -## Becoming a Maintainer - -You can become a maintainer by actively [contributing](CONTRIBUTING.md) to any project, and being nominated by an existing maintainer. - -### Nomination - -Any current maintainer starts a private e-mail thread (until we have a better mechanism, e-mail addresses can usually be found via MAINTAINERS.md + DCO) with all other maintainers on that repository to discuss nomination using the template below. In order to be approved, at least three positive (+1) maintainer votes are necessary, and no vetoes (-1). In rare cases when there are fewer than three maintainers, the positive (+1) votes from all maintainers are required. Any disagreements can be escalated to the repo admin. - -The nomination should clearly identify the person with their real name and a link to their GitHub profile, and the rationale for the nomination, with concrete example contributions. - -### Interest - -Upon receiving at least three positive (+1) maintainer votes, and no vetoes (-1), from existing maintainers after a one week period, the nominating maintainer asks the nominee whether they might be interested in becoming a maintainer on the repository via private e-mail message. - -> This is great work! Based on your valuable contribution and ongoing engagement with the project, the current maintainers invite you to become a co-maintainer for this project. Please respond and let us know if you accept the invitation to become maintainer. - -Individuals accept the nomination by replying, or commenting, for example _"Thank you! I would love to."_ - -### Addition - -Upon receiving three positive (+1) maintainer votes, and no vetoes (-1), from other maintainers, and after having privately confirmed interest with the nominee, the maintainer opens a pull request adding the proposed co-maintainer to MAINTAINERS.md. The pull request is approved and merged. - -> _Content from the above nomination._ -> -> The maintainers have voted and agreed to this nomination. - -The repo admin adjusts the new maintainer’s permissions accordingly, and merges the pull request. - -## Removing a Maintainer - -Removing a maintainer is a disruptive action that the community of maintainers should not undertake lightly. There are several reasons a maintainer will be removed from the project, such as violating the [code of conduct](https://github.com/opensearch-project/.github/blob/main/CODE_OF_CONDUCT.md), or taking other actions that negatively impact the project. - -### Moving On - -There are plenty of reasons that might cause someone to want to take a step back or even a hiatus from a project. Existing maintainers can choose to leave the project at any time, with or without reason, by making a pull request to move themselves to the "Emeritus" section of MAINTAINERS.md, and asking an admin to remove their permissions. - -### Inactivity - -Maintainer status never expires. If a maintainer becomes inactive for a time (usually several months), the repo admin may revoke maintainer level access to the repository for security reasons. A maintainer can reach out to the repo admin to get their permissions reinstated. - -If the repo is left without any maintainers, either by maintainer inactivity or moving on, the repo is considered unmaintained. The repo admin will seek out new maintainers and note the maintenance status in the repo README file. - -### Negative Impact on the Project - -Actions that negatively impact the project will be handled by the admins, in coordination with other maintainers, in balance with the urgency of the issue. Examples would be [Code of Conduct](CODE_OF_CONDUCT.md) violations, deliberate harmful or malicious actions, and security risks. +| Maintainer | GitHub ID | Affiliation | +| --------------- | ----------------------------------------- | ----------- | +| Michael Froh | [msfroh](https://github.com/msfroh) | Amazon | +| Mark Cohen | [macohen](https://github.com/macohen) | Amazon | +| Louis Chu | [noCharger](https://github.com/noCharger) | Amazon | +| Mingshi Liu | [mingshl](https://github.com/mingshl) | Amazon | +| Sean Li | [sejli](https://github.com/sejli) | Amazon | diff --git a/README.md b/README.md index 2733402..06b5ecc 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,30 @@ -[![Build and Test Search Reranker](https://github.com/opensearch-project/search-relevance/actions/workflows/CI.yml/badge.svg)](https://github.com/opensearch-project/search-relevance/actions/workflows/CI.yml) -[![codecov](https://codecov.io/gh/opensearch-project/search-relevance/branch/main/graph/badge.svg?token=PYQO2GW39S)](https://codecov.io/gh/opensearch-project/search-relevance) +[![Build and Test Search Query & Request Transformers](https://github.com/opensearch-project/search-processor/actions/workflows/CI.yml/badge.svg)](https://github.com/opensearch-project/search-processor/actions/workflows/CI.yml) +[![codecov](https://codecov.io/gh/opensearch-project/search-processor/branch/main/graph/badge.svg?token=PYQO2GW39S)](https://codecov.io/gh/opensearch-project/search-processor) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) -# OpenSearch Search Relevance - Reranker +# Search Rerankers: AWS Kendra & AWS Personalize - [Welcome!](#welcome) - [Project Resources](#project-resources) -- [Credits and Acknowledgments](#credits-and-acknowledgments) - [Code of Conduct](#code-of-conduct) - [License](#license) - [Copyright](#copyright) ## Welcome! -This repository currently hosts a plugin to rerank search results before returning them to the client inline. The current plan is to expand this functionality to allow users to add rewriters, rerankers, and log what those components do. +This repository hosts the code for two self-install re-rankers that integrate into [Search Pipelines](https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/). User documentation for the Personalize Reranker is [here](https://opensearch.org/docs/latest/search-plugins/search-pipelines/personalize-search-ranking/). For Kendra, it is [here](https://opensearch.org/docs/latest/search-plugins/search-relevance/index/#reranking-results-with-kendra-intelligent-ranking-for-opensearch). + +# Search Processors: Where Do They Go? +The current guideline for developing processors is that if you are developing a processor that would introduce new dependencies in [OpenSearch Core](https://github.com/opensearch-project/OpenSearch) (e.g. new libraries, makes a network connection outside of OpenSearch), it should be in a separate repository. Please consider creating it in a standalone repository since each processor should be thought of like a \*NIX command with input and output connected by pipes (i.e. a Search Pipeline). Each processor should do one thing and do it well. Otherwise, it could go into the OpenSearch repository under [org.opensearch.search.pipeline.common](https://github.com/opensearch-project/OpenSearch/tree/a08d588691c3b232e65d73b0a0c2fc5c72c870cf/modules/search-pipeline-common). If you have doubts, just create an issue in OpenSearch Core and, if you have one, a new PR. Maintainers will help guide you. + # History This repository has also been used for discussion and ideas around search relevance. These discussions still exist here, however due to the relatively new standard of having one repo per plugin in OpenSearch and our implementations beginning to make it into the OpenSearch build, we have two repositories now. This repository will develop into a plugin that will allow OpenSearch users to rewrite search queries, rerank results, and log data about those actions. The other repository, [dashboards-search-relevance](https://www.github.com/opensearch-projects/dashboards-search-relevance), is where we will build front-end tooling to help relevance engineers and business users tune results. - ## Project Resources -* [Project Website](https://opensearch.org/) -* [Downloads](https://opensearch.org/downloads.html). +* [OpenSearch Project Website](https://opensearch.org/) +* [Downloads](https://opensearch.org/downloads.html) * [Project Principles](https://opensearch.org/#principles) +* [Search Pipelines](https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/) * [Contributing to OpenSearch Search Request Processor](CONTRIBUTING.md) * [Search Relevance](RELEVANCE.md) * [Maintainer Responsibilities](MAINTAINERS.md) diff --git a/RELEVANCE.md b/RELEVANCE.md index 57429ed..333a0c6 100644 --- a/RELEVANCE.md +++ b/RELEVANCE.md @@ -37,7 +37,7 @@ Overall the Relevancy Engineering can be divided into two tiers: 1. Manage business rules, ontologies and manual judgments. 4. Relevancy workbench: 1. Improve algorithms with automated testing, relevance model trainings, personalizations and custom re-rankers. -### [RFC Search Relevance](https://github.com/opensearch-project/search-relevance/issues/1) +### [RFC Search Relevance](https://github.com/opensearch-project/search-processor/issues/1) ## Contributing diff --git a/amazon-kendra-intelligent-ranking/build.gradle b/amazon-kendra-intelligent-ranking/build.gradle new file mode 100644 index 0000000..60821f6 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/build.gradle @@ -0,0 +1,118 @@ +import org.opensearch.gradle.test.RestIntegTestTask + +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'jacoco' + +group = 'org.opensearch' + +def pluginName = 'amazon-kendra-intelligent-ranking' +def pluginDescription = 'Rerank search results using Amazon Kendra Intelligent Ranking' +def projectPath = 'org.opensearch' +def pathToPlugin = 'search.relevance' +def pluginClassName = 'AmazonKendraIntelligentRankingPlugin' + +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}.0" + version "${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') +} + +java { + targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 +} + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + +buildscript { + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + implementation 'com.ibm.icu:icu4j:57.2' + implementation 'org.apache.httpcomponents:httpclient:4.5.14' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2' + implementation 'commons-logging:commons-logging:1.2' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' +} + + +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' + } +} + + +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +testClusters.integTest { + testDistribution = "ARCHIVE" + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) +} + +run { + useCluster testClusters.integTest +} + +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false diff --git a/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java new file mode 100644 index 0000000..ef9df5d --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import org.opensearch.action.support.ActionFilter; +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchPipelinePlugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchResponseProcessor; +import org.opensearch.search.relevance.actionfilter.SearchActionFilter; +import org.opensearch.search.relevance.client.OpenSearchClient; +import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; +import org.opensearch.search.relevance.configuration.SearchConfigurationExtBuilder; +import org.opensearch.search.relevance.transformer.ResultTransformer; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfigurationFactory; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.pipeline.KendraRankingResponseProcessor; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class AmazonKendraIntelligentRankingPlugin extends Plugin implements ActionPlugin, SearchPlugin, SearchPipelinePlugin { + + private OpenSearchClient openSearchClient; + private KendraHttpClient kendraClient; + private KendraIntelligentRanker kendraIntelligentRanker; + private KendraClientSettings kendraClientSettings; + + private Collection getAllResultTransformers() { + // Initialize and add other transformers here + return List.of(this.kendraIntelligentRanker); + } + + private Collection getResultTransformerConfigurationFactories() { + return List.of(KendraIntelligentRankingConfigurationFactory.INSTANCE); + } + + @Override + public List getActionFilters() { + return Arrays.asList(new SearchActionFilter(getAllResultTransformers(), openSearchClient)); + } + + @Override + public List> getSettings() { + // NOTE: cannot use kendraIntelligentRanker.getTransformerSettings because the object is not yet created + List> allTransformerSettings = new ArrayList<>(); + allTransformerSettings.addAll(KendraIntelligentRankerSettings.getAllSettings()); + // Add settings for other transformers here + return allTransformerSettings; + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + this.openSearchClient = new OpenSearchClient(client); + this.kendraClientSettings = KendraClientSettings.getClientSettings(environment.settings()); + this.kendraClient = new KendraHttpClient(this.kendraClientSettings); + this.kendraIntelligentRanker = new KendraIntelligentRanker(this.kendraClient); + + return Arrays.asList( + this.openSearchClient, + this.kendraClientSettings, + this.kendraClient, + this.kendraIntelligentRanker + ); + } + + @Override + public List> getSearchExts() { + Map resultTransformerMap = getResultTransformerConfigurationFactories().stream() + .collect(Collectors.toMap(ResultTransformerConfigurationFactory::getName, i -> i)); + return List.of(new SearchExtSpec<>(SearchConfigurationExtBuilder.NAME, + input -> new SearchConfigurationExtBuilder(input, resultTransformerMap), + parser -> SearchConfigurationExtBuilder.parse(parser, resultTransformerMap))); + } + + @Override + public Map> getResponseProcessors(Parameters parameters) { + return Map.of(KendraRankingResponseProcessor.TYPE, new KendraRankingResponseProcessor.Factory(this.kendraClientSettings)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java similarity index 63% rename from src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java index 1134b35..229172c 100644 --- a/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java @@ -7,35 +7,24 @@ */ package org.opensearch.search.relevance.actionfilter; -import static org.opensearch.action.search.ShardSearchFailure.readShardSearchFailure; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; -import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionResponse; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchResponseSections; -import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.InternalAggregations; @@ -43,27 +32,37 @@ import org.opensearch.search.fetch.subphase.FetchSourceContext; import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.profile.SearchProfileShardResults; -import org.opensearch.search.relevance.configuration.ConfigurationUtils; import org.opensearch.search.relevance.client.OpenSearchClient; +import org.opensearch.search.relevance.configuration.ConfigurationUtils; import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; import org.opensearch.search.relevance.transformer.ResultTransformer; -import org.opensearch.search.relevance.transformer.ResultTransformerType; -import org.opensearch.search.suggest.Suggest; import org.opensearch.tasks.Task; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + public class SearchActionFilter implements ActionFilter { private static final Logger logger = LogManager.getLogger(SearchActionFilter.class); private final int order; private final NamedWriteableRegistry namedWriteableRegistry; - private final Map supportedResultTransformers; + private final Map resultTransformerMap; private final OpenSearchClient openSearchClient; - public SearchActionFilter(Map supportedResultTransformers, OpenSearchClient openSearchClient) { + public SearchActionFilter(Collection supportedResultTransformers, + OpenSearchClient openSearchClient) { order = 10; // TODO: Finalize this value namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList()); - this.supportedResultTransformers = supportedResultTransformers; + resultTransformerMap = supportedResultTransformers.stream() + .collect(Collectors.toMap(t -> t.getConfigurationFactory().getName(), t -> t)); this.openSearchClient = openSearchClient; } @@ -91,7 +90,7 @@ public void app // TODO: Remove originalSearchSource and replace with a deep copy of the SearchRequest object // once https://github.com/opensearch-project/OpenSearch/issues/869 is implemented - SearchSourceBuilder originalSearchSource = null; + SearchSourceBuilder originalSearchSource; if (searchRequest.source() != null) { originalSearchSource = searchRequest.source().shallowCopy(); if (searchRequest.source().fetchSource() != null) { @@ -100,6 +99,8 @@ public void app originalSearchSource.fetchSource(new FetchSourceContext(fetchSourceContext.fetchSource(), fetchSourceContext.includes(), fetchSourceContext.excludes())); } + } else { + originalSearchSource = null; } final String[] indices = searchRequest.indices(); @@ -109,27 +110,29 @@ public void app return; } - List resultTransformerConfigurations = getResultTransformerConfigurations(indices[0], - searchRequest); - LinkedHashMap orderedTransformersAndConfigs = new LinkedHashMap<>(); - for (ResultTransformerConfiguration config : resultTransformerConfigurations) { - ResultTransformer resultTransformer = supportedResultTransformers.get(config.getType()); - // TODO: Should transformers make a decision based on the original request or the request they receive in the chain - if (resultTransformer.shouldTransform(searchRequest, config)) { - searchRequest = resultTransformer.preprocessRequest(searchRequest, config); - orderedTransformersAndConfigs.put(resultTransformer, config); + ActionListener> resultTransformerConfigsListener = ActionListener.wrap(rtc -> { + LinkedHashMap orderedTransformersAndConfigs = new LinkedHashMap<>(); + SearchRequest transformedRequest = searchRequest; + for (ResultTransformerConfiguration config : rtc) { + ResultTransformer resultTransformer = resultTransformerMap.get(config.getTransformerName()); + // TODO: Should transformers make a decision based on the original request or the request they receive in the chain + if (resultTransformer.shouldTransform(searchRequest, config)) { + transformedRequest = resultTransformer.preprocessRequest(transformedRequest, config); + orderedTransformersAndConfigs.put(resultTransformer, config); + } } - } - if (!orderedTransformersAndConfigs.isEmpty()) { - final ActionListener searchResponseListener = createSearchResponseListener( - listener, startTime, orderedTransformersAndConfigs, searchRequest, originalSearchSource); - chain.proceed(task, action, request, searchResponseListener); - return; - } + if (!orderedTransformersAndConfigs.isEmpty()) { + final ActionListener searchResponseListener = createSearchResponseListener( + listener, startTime, orderedTransformersAndConfigs, transformedRequest, originalSearchSource); + chain.proceed(task, action, request, searchResponseListener); + return; + } + chain.proceed(task, action, request, listener); + }, listener::onFailure); - chain.proceed(task, action, request, listener); + getResultTransformerConfigurations(indices[0], searchRequest, resultTransformerConfigsListener); } /** @@ -141,32 +144,32 @@ public void app * @return ordered and validated list of result transformers, empty list if not specified at * either request or index level */ - private List getResultTransformerConfigurations( + private void getResultTransformerConfigurations( final String indexName, - final SearchRequest searchRequest) { + final SearchRequest searchRequest, + ActionListener> resultTransformerConfigListener) { List configs = new ArrayList<>(); // Request level configuration takes precedence over index level configs = ConfigurationUtils.getResultTransformersFromRequestConfiguration(searchRequest); if (!configs.isEmpty()) { - return configs; + resultTransformerConfigListener.onResponse(configs); + return; } // Fetch all index settings for this plugin - String[] settingNames = supportedResultTransformers.values() + String[] settingNames = resultTransformerMap.values() .stream() - .map(t -> t.getTransformerSettings() + .flatMap(t -> t.getTransformerSettings() .stream() - .map(Setting::getKey) - .collect(Collectors.toList())) - .flatMap(Collection::stream) + .map(Setting::getKey)) .toArray(String[]::new); - configs = ConfigurationUtils.getResultTransformersFromIndexConfiguration( - openSearchClient.getIndexSettings(indexName, settingNames)); - return configs; + ActionListener settingsListener = ActionListener.map(resultTransformerConfigListener, + s -> ConfigurationUtils.getResultTransformersFromIndexConfiguration(s, resultTransformerMap)); + openSearchClient.getIndexSettings(indexName, settingNames, settingsListener); } /** @@ -194,21 +197,26 @@ public void onResponse(final Response response) { final SearchResponse searchResponse = (SearchResponse) response; final long totalHits = searchResponse.getHits().getTotalHits().value; if (totalHits == 0) { - logger.info("TotalHits = 0. Returning search response without re-ranking."); + logger.info("TotalHits = 0. Returning search response without transforming."); listener.onResponse(response); return; } logger.debug("Starting re-ranking for search response: {}", searchResponse); try { - final BytesStreamOutput out = new BytesStreamOutput(); - searchResponse.writeTo(out); - - final StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), namedWriteableRegistry); + // Clone search hits (by serializing + deserializing) before transforming + final BytesStreamOutput out = new BytesStreamOutput(); + searchResponse.getHits().writeTo(out); + final StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), + namedWriteableRegistry); SearchHits hits = new SearchHits(in); + for (Map.Entry entry : orderedTransformersAndConfigs.entrySet()) { + long startTime = System.nanoTime(); hits = entry.getKey().transform(hits, searchRequest, entry.getValue()); + long timeTookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + logger.info(entry.getValue().getTransformerName() + ": took " + timeTookMillis + " ms"); } List searchHitsList = Arrays.asList(hits.getHits()); @@ -230,52 +238,22 @@ public void onResponse(final Response response) { } } - // TODO: How to handle SearchHits.TotalHits when transformer modifies the hit count hits = new SearchHits( searchHitsList.toArray(new SearchHit[0]), hits.getTotalHits(), hits.getMaxScore()); - final InternalAggregations aggregations = - in.readBoolean() ? InternalAggregations.readFrom(in) : null; - final Suggest suggest = in.readBoolean() ? new Suggest(in) : null; - final boolean timedOut = in.readBoolean(); - final Boolean terminatedEarly = in.readOptionalBoolean(); - final SearchProfileShardResults profileResults = in.readOptionalWriteable( - SearchProfileShardResults::new); - final int numReducePhases = in.readVInt(); - final SearchResponseSections internalResponse = new InternalSearchResponse(hits, - aggregations, suggest, - profileResults, timedOut, terminatedEarly, numReducePhases); - - final int totalShards = in.readVInt(); - final int successfulShards = in.readVInt(); - final int shardSearchFailureSize = in.readVInt(); - final ShardSearchFailure[] shardFailures; - if (shardSearchFailureSize == 0) { - shardFailures = ShardSearchFailure.EMPTY_ARRAY; - } else { - shardFailures = new ShardSearchFailure[shardSearchFailureSize]; - for (int i = 0; i < shardFailures.length; i++) { - shardFailures[i] = readShardSearchFailure(in); - } - } - - final SearchResponse.Clusters clusters = new SearchResponse.Clusters(in.readVInt(), - in.readVInt(), in.readVInt()); - final String scrollId = in.readOptionalString(); - final int skippedShards = in.readVInt(); - - final long tookInMillis = (System.nanoTime() - startTime) / 1000000; - final SearchResponse newResponse = new SearchResponse(internalResponse, scrollId, - totalShards, successfulShards, - skippedShards, tookInMillis, shardFailures, clusters); + (InternalAggregations) searchResponse.getAggregations(), searchResponse.getSuggest(), + new SearchProfileShardResults(searchResponse.getProfileResults()), searchResponse.isTimedOut(), + searchResponse.isTerminatedEarly(), searchResponse.getNumReducePhases()); + + final long tookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + final SearchResponse newResponse = new SearchResponse(internalResponse, searchResponse.getScrollId(), + searchResponse.getTotalShards(), searchResponse.getSuccessfulShards(), + searchResponse.getSkippedShards(), tookInMillis, searchResponse.getShardFailures(), + searchResponse.getClusters()); listener.onResponse((Response) newResponse); - - // TODO: Change this to a metric - logger.info("Result transformer operations overhead time: {}ms", - tookInMillis - searchResponse.getTook().getMillis()); } catch (final Exception e) { logger.error("Result transformer operations failed.", e); throw new OpenSearchException("Result transformer operations failed.", e); diff --git a/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java similarity index 70% rename from src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java index 1b306dc..42baf89 100644 --- a/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java @@ -12,6 +12,7 @@ import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; public class OpenSearchClient { private final Client client; @@ -20,13 +21,13 @@ public OpenSearchClient(Client client) { this.client = client; } - public Settings getIndexSettings(String indexName, String[] settingNames) { + public void getIndexSettings(String indexName, String[] settingNames, ActionListener settingsListener) { GetSettingsRequest getSettingsRequest = new GetSettingsRequest() .indices(indexName); if (settingNames != null && settingNames.length > 0) { getSettingsRequest.names(settingNames); } - GetSettingsResponse getSettingsResponse = client.execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - return getSettingsResponse.getIndexToSettings().get(indexName); + ActionListener responseListener = ActionListener.map(settingsListener, r -> r.getIndexToSettings().get(indexName)); + client.execute(GetSettingsAction.INSTANCE, getSettingsRequest, responseListener); } } diff --git a/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java new file mode 100644 index 0000000..44081d6 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.configuration; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.search.SearchExtBuilder; +import org.opensearch.search.relevance.transformer.ResultTransformer; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.opensearch.search.relevance.configuration.Constants.RESULT_TRANSFORMER_SETTING_PREFIX; + +public class ConfigurationUtils { + + /** + * Get result transformer configurations from Search Request + * + * @param settings all index settings configured for this plugin + * @param resultTransformerMap map of transformed results + * @return ordered and validated list of result transformers, empty list if not specified + */ + public static List getResultTransformersFromIndexConfiguration(Settings settings, + Map resultTransformerMap) { + List indexLevelConfigs = new ArrayList<>(); + + if (settings != null) { + if (settings.getGroups(RESULT_TRANSFORMER_SETTING_PREFIX) != null) { + for (Map.Entry transformerSettings : settings.getGroups(RESULT_TRANSFORMER_SETTING_PREFIX).entrySet()) { + if (resultTransformerMap.containsKey(transformerSettings.getKey())) { + ResultTransformer transformer = resultTransformerMap.get(transformerSettings.getKey()); + indexLevelConfigs.add(transformer.getConfigurationFactory().configure(transformerSettings.getValue())); + } + } + } + } + + return reorderAndValidateConfigs(indexLevelConfigs); + } + + /** + * Get result transformer configurations from Search Request + * + * @param searchRequest input request + * @return ordered and validated list of result transformers, empty list if not specified + */ + public static List getResultTransformersFromRequestConfiguration( + final SearchRequest searchRequest) { + + // Fetch result transformers specified in request + SearchConfigurationExtBuilder requestLevelSearchConfiguration = null; + if (searchRequest.source() != null && searchRequest.source().ext() != null && !searchRequest.source().ext().isEmpty()) { + // Filter ext builders by name + List extBuilders = searchRequest.source().ext().stream() + .filter(searchExtBuilder -> SearchConfigurationExtBuilder.NAME.equals(searchExtBuilder.getWriteableName())) + .collect(Collectors.toList()); + if (!extBuilders.isEmpty()) { + requestLevelSearchConfiguration = (SearchConfigurationExtBuilder) extBuilders.get(0); + } + } + + List requestLevelConfigs = new ArrayList<>(); + if (requestLevelSearchConfiguration != null) { + requestLevelConfigs = reorderAndValidateConfigs(requestLevelSearchConfiguration.getResultTransformers()); + } + return requestLevelConfigs; + } + + /** + * Sort configurations in ascending order of invocation, and validate + * + * @param configs list of result transformer configurations + * @return ordered and validated list of result transformers + */ + public static List reorderAndValidateConfigs( + final List configs) throws IllegalArgumentException { + + // Sort + configs.sort(Comparator.comparingInt(ResultTransformerConfiguration::getOrder)); + + for (int i = 0; i < configs.size(); ++i) { + if (configs.get(i).getOrder() != (i + 1)) { + throw new IllegalArgumentException("Expected order [" + (i + 1) + "] for transformer [" + + configs.get(i).getTransformerName() + "], but found [" + configs.get(i).getOrder() + "]"); + } + } + + return configs; + } +} diff --git a/src/main/java/org/opensearch/search/relevance/configuration/Constants.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/Constants.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/Constants.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/Constants.java diff --git a/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java similarity index 73% rename from src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java index f63ec46..b52c2f1 100644 --- a/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java @@ -7,10 +7,7 @@ */ package org.opensearch.search.relevance.configuration; -import org.opensearch.search.relevance.transformer.ResultTransformerType; - public abstract class ResultTransformerConfiguration extends TransformerConfiguration { - public abstract ResultTransformerType getType(); - + public abstract String getTransformerName(); } diff --git a/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java new file mode 100644 index 0000000..aece3a7 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.configuration; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; + +public interface ResultTransformerConfigurationFactory { + String getName(); + + /** + * Build configuration based on index settings + * @param indexSettings a set of index settings under a group scoped based on this result transformer's name. + * @return a transformer configuration based on the passed settings. + */ + ResultTransformerConfiguration configure(Settings indexSettings); + + /** + * Build configuration from serialized XContent, e.g. as part of a serialized {@link SearchConfigurationExtBuilder}. + * @param parser an XContentParser pointing to a node serialized from a {@link ResultTransformerConfiguration} of + * this type. + * @return a transformer configuration based on the parameters specified in the XContent. + */ + ResultTransformerConfiguration configure(XContentParser parser) throws IOException; + + /** + * Build configuration from a serialized stream. + * @param streamInput a {@link org.opensearch.core.common.io.stream.Writeable} serialized representation of transformer + * configuration. + * @return configuration the deserialized transformer configuration. + */ + ResultTransformerConfiguration configure(StreamInput streamInput) throws IOException; + +} diff --git a/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java new file mode 100644 index 0000000..53fea0e --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java @@ -0,0 +1,152 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.configuration; + +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchExtBuilder; +import org.opensearch.search.relevance.transformer.TransformerType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.search.relevance.configuration.Constants.SEARCH_CONFIGURATION; + +public class SearchConfigurationExtBuilder extends SearchExtBuilder { + public static final String NAME = SEARCH_CONFIGURATION; + + private static final ParseField RESULT_TRANSFORMER = new ParseField(TransformerType.RESULT_TRANSFORMER.toString()); + + private List resultTransformerConfigurations = new ArrayList<>(); + + public SearchConfigurationExtBuilder() { + } + + public SearchConfigurationExtBuilder(StreamInput input, Map resultTransformerMap) throws IOException { + int numTransformers = input.readInt(); + for (int i = 0; i < numTransformers; i++) { + String transformerName = input.readString(); + ResultTransformerConfigurationFactory transformer = resultTransformerMap.get(transformerName); + if (transformer == null) { + throw new IllegalStateException("Unknown result transformer " + transformerName); + } + resultTransformerConfigurations.add(transformer.configure(input)); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(resultTransformerConfigurations.size()); + for (ResultTransformerConfiguration config : resultTransformerConfigurations) { + out.writeString(config.getTransformerName()); + config.writeTo(out); + } + } + + @Override + public String getWriteableName() { + return NAME; + } + + public static SearchConfigurationExtBuilder parse(XContentParser parser, + Map resultTransformerMap) throws IOException { + SearchConfigurationExtBuilder extBuilder = new SearchConfigurationExtBuilder(); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; + if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) { + throw new ParsingException( + parser.getTokenLocation(), + "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", + parser.getTokenLocation() + ); + } + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (RESULT_TRANSFORMER.match(currentFieldName, parser.getDeprecationHandler())) { + currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (currentFieldName != null) { + if (resultTransformerMap.containsKey(currentFieldName)) { + ResultTransformerConfiguration configuration = + resultTransformerMap.get(currentFieldName).configure(parser); + extBuilder.addResultTransformer(configuration); + } else { + throw new IllegalArgumentException( + "Unrecognized Result Transformer type [" + currentFieldName + "]"); + } + } + } + } else { + throw new IllegalArgumentException("Unrecognized Transformer type [" + currentFieldName + "]"); + } + } else { + throw new ParsingException( + parser.getTokenLocation(), + "Unknown key for a " + token + " in [" + currentFieldName + "].", + parser.getTokenLocation() + ); + } + } + return extBuilder; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(RESULT_TRANSFORMER.getPreferredName()); + for (ResultTransformerConfiguration config : resultTransformerConfigurations) { + builder.field(config.getTransformerName(), config); + } + return builder.endObject(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof SearchConfigurationExtBuilder)) { + return false; + } + SearchConfigurationExtBuilder o = (SearchConfigurationExtBuilder) obj; + HashSet myConfigurations = new HashSet<>(this.resultTransformerConfigurations); + HashSet otherConfigurations = new HashSet<>(o.resultTransformerConfigurations); + return (this.resultTransformerConfigurations.size() == o.resultTransformerConfigurations.size() && + myConfigurations.equals(otherConfigurations)); + } + + @Override + public int hashCode() { + return Objects.hash(this.getClass(), this.resultTransformerConfigurations); + } + + public SearchConfigurationExtBuilder setResultTransformers(final List resultTransformerConfigurations) { + this.resultTransformerConfigurations = resultTransformerConfigurations; + return this; + } + + public List getResultTransformers() { + return this.resultTransformerConfigurations; + } + + public SearchConfigurationExtBuilder addResultTransformer(final ResultTransformerConfiguration resultTransformerConfiguration) { + this.resultTransformerConfigurations.add(resultTransformerConfiguration); + return this; + } +} diff --git a/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java similarity index 84% rename from src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java index 625ab02..5b1099f 100644 --- a/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java @@ -10,9 +10,9 @@ import static org.opensearch.search.relevance.configuration.Constants.ORDER; import static org.opensearch.search.relevance.configuration.Constants.PROPERTIES; -import org.opensearch.common.ParseField; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ToXContentObject; public abstract class TransformerConfiguration implements Writeable, ToXContentObject { protected static final ParseField TRANSFORMER_ORDER = new ParseField(ORDER); diff --git a/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java similarity index 85% rename from src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java index 6568828..72ac0c6 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java @@ -7,20 +7,26 @@ */ package org.opensearch.search.relevance.transformer; -import java.util.List; import org.opensearch.action.search.SearchRequest; import org.opensearch.common.settings.Setting; import org.opensearch.search.SearchHits; import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; -public interface ResultTransformer { +import java.util.List; +public interface ResultTransformer { /** - * Get the list of settings required / supported by the transformer + * Get the list of settings supported by the transformer * @return list of transformer settings */ List> getTransformerSettings(); + /** + * @return a factory able to construct configurations for this transformer. + */ + ResultTransformerConfigurationFactory getConfigurationFactory(); + /** * Decide whether to apply the transformer on the input request * @param request input Search Request diff --git a/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java similarity index 61% rename from src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java index aef6945..67da7bc 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java @@ -21,12 +21,4 @@ public String toString() { return type; } - public static TransformerType fromString(String type) { - for (TransformerType transformerType : values()) { - if (transformerType.type.equalsIgnoreCase(type)) { - return transformerType; - } - } - throw new IllegalArgumentException("Unrecognized Transformer type [" + type + "]"); - } } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java similarity index 86% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java index 458dc55..ef06d03 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java @@ -7,30 +7,20 @@ */ package org.opensearch.search.relevance.transformer.kendraintelligentranking; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.Constants.BODY_FIELD; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.PriorityQueue; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.opensearch.action.search.SearchRequest; import org.opensearch.common.settings.Setting; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchService; import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; import org.opensearch.search.relevance.transformer.ResultTransformer; import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfiguration; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfigurationFactory; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.KendraIntelligentRankingException; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.PassageScore; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.Document; @@ -42,17 +32,29 @@ import org.opensearch.search.relevance.transformer.kendraintelligentranking.preprocess.QueryParser; import org.opensearch.search.relevance.transformer.kendraintelligentranking.preprocess.QueryParser.QueryParserResult; import org.opensearch.search.relevance.transformer.kendraintelligentranking.preprocess.TextTokenizer; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.PriorityQueue; + +import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.Constants.BODY_FIELD; public class KendraIntelligentRanker implements ResultTransformer { - private static final int MAX_SENTENCE_LENGTH_IN_TOKENS = 35; - private static final int MIN_PASSAGE_LENGTH_IN_TOKENS = 100; - private static final int MAX_PASSAGE_COUNT = 10; - private static final int TITLE_TOKENS_TRIMMED = 15; - private static final int BODY_PASSAGE_TRIMMED = 200; - private static final double BM25_B_VALUE = 0.75; - private static final double BM25_K1_VALUE = 1.6; - private static final int TOP_K_PASSAGES = 3; + public static final String NAME = "kendra_intelligent_ranking"; + private static final int MAX_SENTENCE_LENGTH_IN_TOKENS = 35; + private static final int MIN_PASSAGE_LENGTH_IN_TOKENS = 100; + private static final int MAX_PASSAGE_COUNT = 10; + private static final int TITLE_TOKENS_TRIMMED = 15; + private static final int BODY_PASSAGE_TRIMMED = 200; + private static final double BM25_B_VALUE = 0.75; + private static final double BM25_K1_VALUE = 1.6; + private static final int TOP_K_PASSAGES = 3; private static final Logger logger = LogManager.getLogger(KendraIntelligentRanker.class); @@ -71,6 +73,13 @@ public List> getTransformerSettings() { return KendraIntelligentRankerSettings.getAllSettings(); } + + @Override + public ResultTransformerConfigurationFactory getConfigurationFactory() { + return KendraIntelligentRankingConfigurationFactory.INSTANCE; + } + + /** * Check if search request is eligible for rescore * @@ -90,6 +99,10 @@ public boolean shouldTransform(final SearchRequest request, final ResultTransfor request.source().from() >= kendraConfiguration.getProperties().getDocLimit()) { return false; } + if (!kendraClient.isValid()) { + logger.warn("Kendra ranking endpoint was not configured. Skipping reranking."); + return false; + } return true; } @@ -153,7 +166,7 @@ public SearchHits transform(final SearchHits hits, throw new KendraIntelligentRankingException(errorMessage); } List> passages = passageGenerator.generatePassages(docSourceMap.get(bodyFieldName).toString(), - MAX_SENTENCE_LENGTH_IN_TOKENS, MIN_PASSAGE_LENGTH_IN_TOKENS, MAX_PASSAGE_COUNT); + MAX_SENTENCE_LENGTH_IN_TOKENS, MIN_PASSAGE_LENGTH_IN_TOKENS, MAX_PASSAGE_COUNT); List> topPassages = getTopPassages(queryParserResult.getQueryText(), passages); List tokenizedTitle = null; if (titleFieldName != null && docSourceMap.get(titleFieldName) != null) { @@ -172,10 +185,10 @@ public SearchHits transform(final SearchHits hits, } originalHitsAsDocuments.add(new Document( originalHits.get(j).getId() + "@" + (i + 1), - originalHits.get(j).getId(), - tokenizedTitle, - passageTokens, - originalHits.get(j).getScore())); + originalHits.get(j).getId(), + tokenizedTitle, + passageTokens, + originalHits.get(j).getScore())); } // Map search hits by their ID in order to map Kendra response documents back to hits later idToSearchHitMap.put(originalHits.get(j).getId(), originalHits.get(j)); @@ -194,16 +207,16 @@ public SearchHits transform(final SearchHits hits, rescoreResultItem.getDocumentId()); logger.error(errorMessage); throw new KendraIntelligentRankingException(errorMessage); - } - searchHit.score(rescoreResultItem.getScore()); - maxScore = Math.max(maxScore, rescoreResultItem.getScore()); - newSearchHits.add(searchHit); + } + searchHit.score(rescoreResultItem.getScore()); + maxScore = Math.max(maxScore, rescoreResultItem.getScore()); + newSearchHits.add(searchHit); } // Add remaining hits to response, which are already sorted by OpenSearch score for (int i = numberOfHitsToRerank; i < originalHits.size(); ++i) { - newSearchHits.add(originalHits.get(i)); + newSearchHits.add(originalHits.get(i)); } - return new SearchHits(newSearchHits.toArray(new SearchHit[newSearchHits.size()]), hits.getTotalHits(), maxScore); + return new SearchHits(newSearchHits.toArray(new SearchHit[0]), hits.getTotalHits(), maxScore); } catch (Exception ex) { logger.error("Failed to rescore. Returning original search results without rescore.", ex); return hits; @@ -213,7 +226,7 @@ public SearchHits transform(final SearchHits hits, private List> getTopPassages(final String queryText, final List> passages) { List query = textTokenizer.tokenize(queryText); BM25Scorer bm25Scorer = new BM25Scorer(BM25_B_VALUE, BM25_K1_VALUE, passages); - PriorityQueue pq = new PriorityQueue<>(Comparator.comparingDouble(x -> x.getScore())); + PriorityQueue pq = new PriorityQueue<>(Comparator.comparingDouble(PassageScore::getScore)); for (int i = 0; i < passages.size(); i++) { double score = bm25Scorer.score(query, passages.get(i)); diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java similarity index 98% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java index 44fb113..523aa99 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java @@ -21,7 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java similarity index 58% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java index d30fb27..11b44dd 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java @@ -29,15 +29,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; +import java.io.Closeable; import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; +import org.opensearch.core.common.Strings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreRequest; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreResult; -public class KendraHttpClient { +public class KendraHttpClient implements Closeable { private static final String KENDRA_RANKING_SERVICE_NAME = "kendra-ranking"; private static final String KENDRA_RESCORE_URI = "rescore"; private static final String KENDRA_RESCORE_EXECUTION_PLANS = "rescore-execution-plans"; @@ -50,48 +52,58 @@ public class KendraHttpClient { private final AWS4Signer aws4Signer; private final String serviceEndpoint; private final String executionPlanId; - private final ObjectMapper objectMapper; + private final ObjectMapper objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + @SuppressWarnings("removal") public KendraHttpClient(KendraClientSettings clientSettings) { - amazonHttpClient = AccessController.doPrivileged((PrivilegedAction) () -> new AmazonHttpClient(new ClientConfiguration())); - errorHandler = new SimpleAwsErrorHandler(); - responseHandler = new SimpleResponseHandler(); - aws4Signer = new AWS4Signer(); - aws4Signer.setServiceName(KENDRA_RANKING_SERVICE_NAME); - aws4Signer.setRegionName(clientSettings.getServiceRegion()); - objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); serviceEndpoint = clientSettings.getServiceEndpoint(); executionPlanId = clientSettings.getExecutionPlanId(); + if (isValid()) { + amazonHttpClient = AccessController.doPrivileged((PrivilegedAction) () -> new AmazonHttpClient(new ClientConfiguration())); + errorHandler = new SimpleAwsErrorHandler(); + responseHandler = new SimpleResponseHandler(); + aws4Signer = new AWS4Signer(); + aws4Signer.setServiceName(KENDRA_RANKING_SERVICE_NAME); + aws4Signer.setRegionName(clientSettings.getServiceRegion()); - final AWSCredentialsProvider credentialsProvider; - final AWSCredentials credentials = clientSettings.getCredentials(); - if (credentials == null) { - // Use environment variables, system properties or instance profile credentials. - credentialsProvider = DefaultAWSCredentialsProviderChain.getInstance(); - } else { - // Use keystore credentials. - credentialsProvider = new AWSStaticCredentialsProvider(credentials); - } + final AWSCredentialsProvider credentialsProvider; + final AWSCredentials credentials = clientSettings.getCredentials(); + if (credentials == null) { + // Use environment variables, system properties or instance profile credentials. + credentialsProvider = DefaultAWSCredentialsProviderChain.getInstance(); + } else { + // Use keystore credentials. + credentialsProvider = new AWSStaticCredentialsProvider(credentials); + } - final String assumeRoleArn = clientSettings.getAssumeRoleArn(); - if (assumeRoleArn != null && !assumeRoleArn.isBlank()) { - // If AssumeRoleArn was provided in config, use auto-refreshed role credentials. - awsCredentialsProvider = AccessController.doPrivileged( - (PrivilegedAction) () -> { - AWSSecurityTokenService awsSecurityTokenService = AWSSecurityTokenServiceClientBuilder.standard() - .withCredentials(credentialsProvider) - .withRegion(clientSettings.getServiceRegion()) - .build(); - - return new STSAssumeRoleSessionCredentialsProvider.Builder(clientSettings.getAssumeRoleArn(), ASSUME_ROLE_SESSION_NAME) - .withStsClient(awsSecurityTokenService) - .build(); - }); + final String assumeRoleArn = clientSettings.getAssumeRoleArn(); + if (assumeRoleArn != null && !assumeRoleArn.isBlank()) { + // If AssumeRoleArn was provided in config, use auto-refreshed role credentials. + awsCredentialsProvider = AccessController.doPrivileged( + (PrivilegedAction) () -> { + AWSSecurityTokenService awsSecurityTokenService = AWSSecurityTokenServiceClientBuilder.standard() + .withCredentials(credentialsProvider) + .withRegion(clientSettings.getServiceRegion()) + .build(); + + return new STSAssumeRoleSessionCredentialsProvider.Builder(clientSettings.getAssumeRoleArn(), ASSUME_ROLE_SESSION_NAME) + .withStsClient(awsSecurityTokenService) + .build(); + }); + } else { + awsCredentialsProvider = credentialsProvider; + } } else { - awsCredentialsProvider = credentialsProvider; + amazonHttpClient = null; + aws4Signer = null; + awsCredentialsProvider = null; + errorHandler = null; + responseHandler = null; } } + @SuppressWarnings({ "deprecation", "removal" }) public RescoreResult rescore(RescoreRequest rescoreRequest) { return AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -119,4 +131,15 @@ public URI buildRescoreURI() { return URI.create(String.join("/", serviceEndpoint, KENDRA_RESCORE_EXECUTION_PLANS, executionPlanId, KENDRA_RESCORE_URI)); } + + public boolean isValid() { + return !Strings.isNullOrEmpty(serviceEndpoint) && !Strings.isNullOrEmpty(executionPlanId); + } + + @Override + public void close() { + if (amazonHttpClient != null) { + amazonHttpClient.shutdown(); + } + } } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java similarity index 96% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java index 1648ad9..062a221 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java @@ -12,7 +12,7 @@ import java.util.List; import java.util.function.Function; import org.opensearch.common.settings.SecureSetting; -import org.opensearch.common.settings.SecureString; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; @@ -29,7 +29,7 @@ public class KendraIntelligentRankerSettings { */ static final class FieldValidator implements Setting.Validator> { - private String settingName; + private final String settingName; public FieldValidator(final String name) { this.settingName = name; @@ -48,7 +48,7 @@ public void validate(List value) { */ static final class DocLimitValidator implements Setting.Validator { - private String settingName; + private final String settingName; public DocLimitValidator(final String name) { this.settingName = name; @@ -115,7 +115,7 @@ public void validate(Integer value) { public static final Setting ASSUME_ROLE_ARN_SETTING = Setting.simpleString("kendra_intelligent_ranking.service.assume_role_arn", Setting.Property.NodeScope); - public static final List> getAllSettings() { + public static List> getAllSettings() { return Arrays.asList( KENDRA_ORDER_SETTING, KENDRA_BODY_FIELD_SETTING, diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java similarity index 85% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java index 6bafb3d..2842d11 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java @@ -7,39 +7,40 @@ */ package org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration; -import static org.opensearch.search.relevance.configuration.Constants.ORDER; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.Constants.DOC_LIMIT_SETTING_NAME; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.Constants.KENDRA_DEFAULT_DOC_LIMIT; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.BODY_FIELD_VALIDATOR; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.DOC_LIMIT_VALIDATOR; -import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.TITLE_FIELD_VALIDATOR; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.configuration.TransformerConfiguration; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Objects; -import org.opensearch.common.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.ObjectParser; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; -import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; -import org.opensearch.search.relevance.configuration.TransformerConfiguration; -import org.opensearch.search.relevance.transformer.ResultTransformerType; + +import static org.opensearch.search.relevance.configuration.Constants.ORDER; +import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.Constants.KENDRA_DEFAULT_DOC_LIMIT; +import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.BODY_FIELD_VALIDATOR; +import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.DOC_LIMIT_VALIDATOR; +import static org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings.TITLE_FIELD_VALIDATOR; public class KendraIntelligentRankingConfiguration extends ResultTransformerConfiguration { private static final ObjectParser PARSER; static { - PARSER = new ObjectParser("kendra_intelligent_ranking_configuration", KendraIntelligentRankingConfiguration::new); + PARSER = new ObjectParser<>("kendra_intelligent_ranking_configuration", KendraIntelligentRankingConfiguration::new); PARSER.declareInt(TransformerConfiguration::setOrder, TRANSFORMER_ORDER); PARSER.declareObject(KendraIntelligentRankingConfiguration::setProperties, - (p, c) -> KendraIntelligentRankingProperties.parse(p, c), + KendraIntelligentRankingProperties::parse, TRANSFORMER_PROPERTIES); } @@ -66,8 +67,8 @@ public KendraIntelligentRankingConfiguration(Settings settings) { } @Override - public ResultTransformerType getType() { - return ResultTransformerType.KENDRA_INTELLIGENT_RANKING; + public String getTransformerName() { + return KendraIntelligentRanker.NAME; } @Override @@ -76,7 +77,7 @@ public void writeTo(StreamOutput out) throws IOException { this.properties.writeTo(out); } - public static ResultTransformerConfiguration parse(XContentParser parser, Void context) throws IOException { + public static KendraIntelligentRankingConfiguration parse(XContentParser parser) throws IOException { try { KendraIntelligentRankingConfiguration configuration = PARSER.parse(parser, null); if (configuration != null && configuration.getOrder() <= 0) { @@ -90,7 +91,7 @@ public static ResultTransformerConfiguration parse(XContentParser parser, Void c } @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); builder.field(TRANSFORMER_ORDER.getPreferredName(), this.order); builder.field(TRANSFORMER_PROPERTIES.getPreferredName(), this.properties); @@ -154,7 +155,7 @@ public KendraIntelligentRankingProperties(final List bodyFields, public KendraIntelligentRankingProperties(StreamInput input) throws IOException { this.bodyFields = input.readStringList(); - this.bodyFields = input.readStringList(); + this.titleFields = input.readStringList(); this.docLimit = input.readInt(); } @@ -195,8 +196,8 @@ public boolean equals(Object o) { KendraIntelligentRankingProperties properties = (KendraIntelligentRankingProperties) o; - return (bodyFields == properties.bodyFields) && (titleFields == properties.titleFields) && - (docLimit == properties.docLimit); + return bodyFields.equals(properties.bodyFields) && titleFields.equals(properties.titleFields) && + docLimit == properties.docLimit; } @Override @@ -227,5 +228,6 @@ public int getDocLimit() { public void setDocLimit(final int docLimit) { this.docLimit = docLimit; } + } } diff --git a/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java new file mode 100644 index 0000000..080a8c2 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; + +import java.io.IOException; + +public class KendraIntelligentRankingConfigurationFactory implements ResultTransformerConfigurationFactory { + private KendraIntelligentRankingConfigurationFactory() { + } + + public static final KendraIntelligentRankingConfigurationFactory INSTANCE = + new KendraIntelligentRankingConfigurationFactory(); + + @Override + public String getName() { + return KendraIntelligentRanker.NAME; + } + + @Override + public ResultTransformerConfiguration configure(Settings indexSettings) { + return new KendraIntelligentRankingConfiguration(indexSettings); + } + + @Override + public ResultTransformerConfiguration configure(XContentParser parser) throws IOException { + return KendraIntelligentRankingConfiguration.parse(parser); + } + + @Override + public ResultTransformerConfiguration configure(StreamInput streamInput) throws IOException { + return new KendraIntelligentRankingConfiguration(streamInput); + } +} diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java similarity index 68% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java index d50b840..703d32a 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java @@ -9,7 +9,7 @@ import java.io.IOException; import org.opensearch.OpenSearchException; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; public class KendraIntelligentRankingException extends OpenSearchException { public KendraIntelligentRankingException(StreamInput in) throws IOException { @@ -20,11 +20,4 @@ public KendraIntelligentRankingException(String message) { super(message); } - public KendraIntelligentRankingException(String message, Throwable cause) { - super(message, cause); - } - - public KendraIntelligentRankingException(String message, Throwable cause, Object... args) { - super(message, cause, args); - } } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java similarity index 72% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java index 3328b7a..d57d004 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.annotation.JsonNaming; import java.util.List; +import java.util.Objects; @JsonNaming(PropertyNamingStrategies.UpperCamelCaseStrategy.class) public class Document { @@ -30,6 +31,12 @@ public Document(String id, String groupId, List tokenizedTitle, List titleField; + private final List bodyField; + private final int docLimit; + private final String tag; + private final String description; + private final KendraHttpClient kendraClient; + + private static final Logger logger = LogManager.getLogger(KendraRankingResponseProcessor.class); + + /** + * Constructor that apply configuration for kendra re-ranking + * + * @param tag processor tag + * @param description processor description + * @param ignoreFailure processor ignoreFailure config + * @param titleField titleField applied to kendra re-ranking + * @param bodyField bodyField applied to kendra re-ranking + * @param inputDocLimit docLimit applied to kendra re-ranking + * @param kendraClient kendraClient to connect with kendra + */ + public KendraRankingResponseProcessor(String tag, String description, boolean ignoreFailure, List titleField, List bodyField, Integer inputDocLimit, KendraHttpClient kendraClient) { + super(tag, description, ignoreFailure); + this.titleField = titleField; + this.bodyField = bodyField; + this.tag = tag; + this.description = description; + this.kendraClient = kendraClient; + int docLimit; + if (inputDocLimit == null) { + docLimit = KENDRA_DEFAULT_DOC_LIMIT; + } else { + docLimit = inputDocLimit; + } + this.docLimit = docLimit; + } + + /** + * Gets the type of the processor. + */ + @Override + public String getType() { + return TYPE; + } + + /** + * Gets the tag of a processor. + */ + @Override + public String getTag() { + return tag; + } + + /** + * Gets the description of a processor. + */ + @Override + public String getDescription() { + return description; + } + + + /** + * Transform the response hit and apply kendra re-ranking logic + */ + @Override + public SearchResponse processResponse(SearchRequest request, SearchResponse response) throws Exception { + SearchHits hits = response.getHits(); + + if (hits.getHits().length == 0) { + // Avoid call to re-rank empty results + logger.info("TotalHits = 0. Returning search response without transforming."); + return response; + } + + KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties properties = new KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties(bodyField, titleField, docLimit); + KendraIntelligentRankingConfiguration configuration = new KendraIntelligentRankingConfiguration(1, properties); + KendraIntelligentRanker ranker = new KendraIntelligentRanker(this.kendraClient); + SearchRequest processedRequest = ranker.preprocessRequest(request, configuration); + + if (ranker.shouldTransform(processedRequest, configuration)) { + long startTime = System.nanoTime(); + SearchHits reRankedSearchHits = ranker.transform(hits, processedRequest, configuration); + long timeTookMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + + final SearchResponseSections internalResponse = new InternalSearchResponse(reRankedSearchHits, + (InternalAggregations) response.getAggregations(), response.getSuggest(), + new SearchProfileShardResults(response.getProfileResults()), response.isTimedOut(), + response.isTerminatedEarly(), response.getNumReducePhases()); + + final SearchResponse newResponse = new SearchResponse(internalResponse, response.getScrollId(), + response.getTotalShards(), response.getSuccessfulShards(), + response.getSkippedShards(), timeTookMillis, response.getShardFailures(), + response.getClusters()); + logger.info("kendra ranking processor took " + timeTookMillis + " ms"); + return newResponse; + } else + return response; + } + + /** + * This is a factor that creates the KendraRankingResponseProcessor + */ + public static final class Factory implements Processor.Factory { + + private final KendraClientSettings clientSettings; + + /** + * Constructor for factory + * @param kendraClientSettings credentials to create kendra client + */ + public Factory(KendraClientSettings kendraClientSettings) { + this.clientSettings = kendraClientSettings; + } + + public KendraRankingResponseProcessor create( + Map> processorFactories, + String tag, + String description, + boolean ignoreFailure, + Map config, + PipelineContext pipelineContext + ) throws Exception { + List titleField = Collections.singletonList(ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "title_field")); + List bodyField = Collections.singletonList(ConfigurationUtils.readStringProperty(TYPE, tag, config, "body_field")); + String inputDocLimit = ConfigurationUtils.readOptionalStringOrIntProperty(TYPE, tag, config, "doc_limit"); + KendraHttpClient kendraClient = new KendraHttpClient(this.clientSettings); + int docLimit; + if (inputDocLimit == null) { + docLimit = KENDRA_DEFAULT_DOC_LIMIT; + } else { + docLimit = Integer.parseInt(inputDocLimit); + } + return new KendraRankingResponseProcessor(tag, description, ignoreFailure, titleField, bodyField, docLimit, kendraClient); + } + } +} diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java diff --git a/src/main/plugin-metadata/plugin-security.policy b/amazon-kendra-intelligent-ranking/src/main/plugin-metadata/plugin-security.policy similarity index 100% rename from src/main/plugin-metadata/plugin-security.policy rename to amazon-kendra-intelligent-ranking/src/main/plugin-metadata/plugin-security.policy diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java new file mode 100644 index 0000000..cc13e70 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import java.io.IOException; + +public class AmazonKendraIntelligentRankingPluginIT extends OpenSearchRestTestCase { + + public void testPluginInstalled() throws IOException, ParseException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins")); + String body = EntityUtils.toString(response.getEntity()); + + logger.info("response body: {}", body); + assertNotNull(body); + assertTrue(body.contains("amazon-kendra-intelligent-ranking")); + } +} diff --git a/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java diff --git a/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java similarity index 89% rename from src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java index 7d748f5..496adbf 100644 --- a/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java @@ -9,8 +9,6 @@ import org.apache.lucene.search.TotalHits; import org.mockito.Mockito; -import org.opensearch.action.ActionFuture; -import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -25,23 +23,25 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.client.Client; -import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.collect.ImmutableOpenMap; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.internal.InternalSearchResponse; import org.opensearch.search.relevance.client.OpenSearchClient; import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; import org.opensearch.search.relevance.configuration.SearchConfigurationExtBuilder; import org.opensearch.search.relevance.transformer.ResultTransformer; -import org.opensearch.search.relevance.transformer.ResultTransformerType; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; @@ -55,7 +55,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; public class SearchActionFilterTests extends OpenSearchTestCase { @@ -65,7 +66,7 @@ public class SearchActionFilterTests extends OpenSearchTestCase { public void testIgnoresDelete() { Client client = Mockito.mock(Client.class); OpenSearchClient openSearchClient = new OpenSearchClient(client); - SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyMap(), openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyList(), openSearchClient); Task task = Mockito.mock(Task.class); DeleteRequest deleteRequest = new DeleteRequestBuilder(null, DeleteAction.INSTANCE).request(); @@ -82,7 +83,7 @@ public void testIgnoresDelete() { public void testIgnoresSearchRequestOnZeroIndices() { Client client = Mockito.mock(Client.class); OpenSearchClient openSearchClient = new OpenSearchClient(client); - SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyMap(), openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyList(), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE).request(); @@ -99,7 +100,7 @@ public void testIgnoresSearchRequestOnZeroIndices() { public void testIgnoresSearchRequestOnMultipleIndices() { Client client = Mockito.mock(Client.class); OpenSearchClient openSearchClient = new OpenSearchClient(client); - SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyMap(), openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyList(), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -114,21 +115,19 @@ public void testIgnoresSearchRequestOnMultipleIndices() { private static Client buildMockClient(String indexName, Settings... settings) { Client client = Mockito.mock(Client.class); - ActionFuture mockGetSettingsFuture = Mockito.mock(ActionFuture.class); Settings.Builder settingsBuilder = Settings.builder(); for (Settings settingsEntry : settings) { settingsBuilder.put(settingsEntry); } Settings settingsObj = settingsBuilder.build(); - ImmutableOpenMap indexSettingsMap = ImmutableOpenMap.builder() - .fPut(indexName, settingsObj) - .build(); - ImmutableOpenMap emptyMap = ImmutableOpenMap.builder().build(); - GetSettingsResponse getSettingsResponse = new GetSettingsResponse(indexSettingsMap, emptyMap); - when(mockGetSettingsFuture.actionGet()).thenReturn(getSettingsResponse); - when(client.execute(eq(GetSettingsAction.INSTANCE), any(GetSettingsRequest.class))) - .thenReturn(mockGetSettingsFuture); + Map indexSettingsMap = Map.of(indexName, settingsObj); + GetSettingsResponse getSettingsResponse = new GetSettingsResponse(indexSettingsMap, Collections.emptyMap()); + doAnswer(invocation -> { + ActionListener responseListener = invocation.getArgument(2); + responseListener.onResponse(getSettingsResponse); + return null; + }).when(client).execute(eq(GetSettingsAction.INSTANCE), any(GetSettingsRequest.class), any(ActionListener.class)); return client; } @@ -138,7 +137,7 @@ private static Client buildMockClient(String indexName, Settings... settings) { public void testOperatesOnSingleIndexWithNoTransformers() { Client client = buildMockClient("index"); OpenSearchClient openSearchClient = new OpenSearchClient(client); - SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyMap(), openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(Collections.emptyList(), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -147,12 +146,16 @@ public void testOperatesOnSingleIndexWithNoTransformers() { AtomicBoolean proceedCalled = new AtomicBoolean(false); ActionFilterChain searchFilterChain = (task1, action, request, listener) -> proceedCalled.set(true); - searchActionFilter.apply(task, SearchAction.NAME, searchRequest, null, searchFilterChain); + ActionListener mockListener = mock(ActionListener.class); + searchActionFilter.apply(task, SearchAction.NAME, searchRequest, mockListener, searchFilterChain); assertTrue(proceedCalled.get()); } private static class MockTransformer implements ResultTransformer { + + public static final String NAME = "mock_transformer"; + public MockTransformer() { requestTransformer = i -> {}; } @@ -167,13 +170,18 @@ public MockTransformer(Consumer requestTransformer) { private boolean transformWasCalled = false; private boolean preproccessRequestWasCalled = false; - @Override public List> getTransformerSettings() { getTransformerSettingsWasCalled = true; return Collections.emptyList(); } + @Override + public ResultTransformerConfigurationFactory getConfigurationFactory() { + return MOCK_CONFIGURATION_FACTORY; + } + + @Override public boolean shouldTransform(SearchRequest request, ResultTransformerConfiguration configuration) { shouldTransformWasCalled = true; @@ -202,9 +210,8 @@ public int getOrder() { } @Override - public ResultTransformerType getType() { - // For now, the only supported type - return ResultTransformerType.KENDRA_INTELLIGENT_RANKING; + public String getTransformerName() { + return MockTransformer.NAME; } @Override @@ -217,6 +224,28 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } }; + private static ResultTransformerConfigurationFactory MOCK_CONFIGURATION_FACTORY = new ResultTransformerConfigurationFactory() { + @Override + public String getName() { + return MockTransformer.NAME; + } + + @Override + public ResultTransformerConfiguration configure(Settings indexSettings) { + return MOCK_TRANSFORMER_CONFIGURATION; + } + + @Override + public ResultTransformerConfiguration configure(XContentParser parser) { + return MOCK_TRANSFORMER_CONFIGURATION; + } + + @Override + public ResultTransformerConfiguration configure(StreamInput streamInput) { + return MOCK_TRANSFORMER_CONFIGURATION; + } + }; + /** * Even if a transformer is wired into the SearchActionFilter, if it's not enabled by search request or * index setting, the transformer will not be called. @@ -227,10 +256,7 @@ public void testTransformerDoesNotRunWhenNotEnabled() { MockTransformer mockTransformer = new MockTransformer(); - Map transformerMap = - Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, mockTransformer); - - SearchActionFilter searchActionFilter = new SearchActionFilter(transformerMap, openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(List.of(mockTransformer), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -239,7 +265,8 @@ public void testTransformerDoesNotRunWhenNotEnabled() { AtomicBoolean proceedCalled = new AtomicBoolean(false); ActionFilterChain searchFilterChain = (task1, action, request, listener) -> proceedCalled.set(true); - searchActionFilter.apply(task, SearchAction.NAME, searchRequest, null, searchFilterChain); + ActionListener mockListener = mock(ActionListener.class); + searchActionFilter.apply(task, SearchAction.NAME, searchRequest, mockListener, searchFilterChain); assertTrue(proceedCalled.get()); // We should try to check for index-level settings assertTrue(mockTransformer.getTransformerSettingsWasCalled); @@ -257,10 +284,7 @@ public void testTransformEnabledInRequest() throws IOException { MockTransformer mockTransformer = new MockTransformer(); - Map transformerMap = - Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, mockTransformer); - - SearchActionFilter searchActionFilter = new SearchActionFilter(transformerMap, openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(List.of(mockTransformer), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -332,7 +356,7 @@ private static SearchResponse buildMockSearchResponse(int numHits) throws IOExce */ public void testTransformEnabledByIndexSetting() throws IOException { String prefix = "index.plugin.searchrelevance.result_transformer." + - ResultTransformerType.KENDRA_INTELLIGENT_RANKING; + MockTransformer.NAME; Settings enablePluginSettings = Settings.builder() .put(prefix + ".order", 1) .build(); @@ -341,10 +365,7 @@ public void testTransformEnabledByIndexSetting() throws IOException { MockTransformer mockTransformer = new MockTransformer(); - Map transformerMap = - Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, mockTransformer); - - SearchActionFilter searchActionFilter = new SearchActionFilter(transformerMap, openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(List.of(mockTransformer), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -398,10 +419,7 @@ public void testOutputUsesOriginalSourceParameters() throws IOException { .fetchSource(true); }); - Map transformerMap = - Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, mockTransformer); - - SearchActionFilter searchActionFilter = new SearchActionFilter(transformerMap, openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(List.of(mockTransformer), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) @@ -476,10 +494,7 @@ public void testReturnEmptyWhenOriginalFromExceedsHitCount() throws IOException .fetchSource(true); }); - Map transformerMap = - Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, mockTransformer); - - SearchActionFilter searchActionFilter = new SearchActionFilter(transformerMap, openSearchClient); + SearchActionFilter searchActionFilter = new SearchActionFilter(List.of(mockTransformer), openSearchClient); Task task = Mockito.mock(Task.class); SearchRequest searchRequest = new SearchRequestBuilder(null, SearchAction.INSTANCE) diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java new file mode 100644 index 0000000..362a5fd --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.configuration; + +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class SearchConfigurationExtBuilderTests extends OpenSearchTestCase { + private static final String TRANSFORMER_NAME = "mock_transformer"; + + private static class MockResultTransformerConfiguration extends ResultTransformerConfiguration { + private final String configuredValue; + + public MockResultTransformerConfiguration(String configuredValue) { + this.configuredValue = configuredValue; + } + + @Override + public String getTransformerName() { + return TRANSFORMER_NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(configuredValue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("configuredValue", configuredValue); + return builder.endObject(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MockResultTransformerConfiguration that = (MockResultTransformerConfiguration) o; + return configuredValue.equals(that.configuredValue); + } + + @Override + public int hashCode() { + return Objects.hash(configuredValue); + } + + @Override + public String toString() { + return "MockResultTransformerConfiguration{" + + "configuredValue='" + configuredValue + '\'' + + '}'; + } + } + + private static ResultTransformerConfigurationFactory MOCK_RESULT_TRANSFORMER_CONFIGURATION_FACTORY = new ResultTransformerConfigurationFactory() { + @Override + public String getName() { + return TRANSFORMER_NAME; + } + + @Override + public ResultTransformerConfiguration configure(Settings indexSettings) { + return null; + } + + @Override + public ResultTransformerConfiguration configure(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + assertSame(XContentParser.Token.FIELD_NAME, token); + assertEquals("configuredValue", parser.currentName()); + token = parser.nextToken(); + assertSame(XContentParser.Token.VALUE_STRING, token); + String configuredValue = parser.text(); + return new MockResultTransformerConfiguration(configuredValue); + } + + @Override + public ResultTransformerConfiguration configure(StreamInput streamInput) throws IOException { + return new MockResultTransformerConfiguration(streamInput.readString()); + } + }; + public static final Map RESULT_TRANSFORMER_CONFIGURATION_FACTORY_MAP = Map.of(TRANSFORMER_NAME, MOCK_RESULT_TRANSFORMER_CONFIGURATION_FACTORY); + + + public void testXContentRoundTrip() throws IOException { + MockResultTransformerConfiguration configuration = new MockResultTransformerConfiguration(randomUnicodeOfLength(10)); + SearchConfigurationExtBuilder searchConfigurationExtBuilder = new SearchConfigurationExtBuilder() + .addResultTransformer(configuration); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference serialized = XContentHelper.toXContent(searchConfigurationExtBuilder, xContentType, true); + + XContentParser parser = createParser(xContentType.xContent(), serialized); + + SearchConfigurationExtBuilder deserialized = + SearchConfigurationExtBuilder.parse(parser, RESULT_TRANSFORMER_CONFIGURATION_FACTORY_MAP); + assertEquals(searchConfigurationExtBuilder, deserialized); + } + + public void testStreamRoundTrip() throws IOException { + MockResultTransformerConfiguration configuration = new MockResultTransformerConfiguration(randomUnicodeOfLength(10)); + SearchConfigurationExtBuilder searchConfigurationExtBuilder = new SearchConfigurationExtBuilder() + .addResultTransformer(configuration); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + searchConfigurationExtBuilder.writeTo(bytesStreamOutput); + + SearchConfigurationExtBuilder deserialized = new SearchConfigurationExtBuilder(bytesStreamOutput.bytes().streamInput(), + RESULT_TRANSFORMER_CONFIGURATION_FACTORY_MAP); + assertEquals(searchConfigurationExtBuilder, deserialized); + } +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java similarity index 69% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java index d975f07..1d176d8 100644 --- a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java @@ -8,47 +8,38 @@ package org.opensearch.search.relevance.transformer.kendraintelligentranking; import org.apache.lucene.search.TotalHits; -import org.mockito.Mockito; import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.settings.Setting; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.relevance.configuration.ResultTransformerConfiguration; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraIntelligentClientTests; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfiguration; import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreRequest; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreResult; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreResultItem; -import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; import java.util.stream.Collectors; -public class KendraIntelligentRankerTests extends OpenSearchTestCase { - private static KendraHttpClient buildMockHttpClient(Function mockRescoreImpl) { - KendraHttpClient kendraHttpClient = Mockito.mock(KendraHttpClient.class); - Mockito.doAnswer(invocation -> { - RescoreRequest rescoreRequest = invocation.getArgument(0); - return mockRescoreImpl.apply(rescoreRequest); - }).when(kendraHttpClient).rescore(Mockito.any(RescoreRequest.class)); - return kendraHttpClient; - } - - private static KendraHttpClient buildMockHttpClient() { - return buildMockHttpClient(r -> new RescoreResult()); - } +public class KendraIntelligentRankerTests extends KendraIntelligentClientTests { public void testGetSettings() { List> settings = new KendraIntelligentRanker(buildMockHttpClient()).getTransformerSettings(); @@ -105,6 +96,38 @@ public void testShouldNotTransformWithSort() { assertFalse(shouldTransform); } + public void testShouldNotTransformWithInvalidClient() { + Settings emptySettings = Settings.builder().build(); + KendraHttpClient emptyClient = new KendraHttpClient(KendraClientSettings.getClientSettings(emptySettings)); + testWithInvalidClient(emptyClient); + + Settings settingsWithExecutionPlan = Settings.builder() + .put(KendraIntelligentRankerSettings.EXECUTION_PLAN_ID_SETTING.getKey(), "foo-plan") + .build(); + testWithInvalidClient(new KendraHttpClient(KendraClientSettings.getClientSettings(settingsWithExecutionPlan))); + + Settings settingsWithEndpoint = Settings.builder() + .put(KendraIntelligentRankerSettings.SERVICE_ENDPOINT_SETTING.getKey(), + "https://kendra-ranking.us-west-2.api.aws") + .build(); + testWithInvalidClient(new KendraHttpClient(KendraClientSettings.getClientSettings(settingsWithEndpoint))); + } + + private void testWithInvalidClient(KendraHttpClient invalidClient) { + KendraIntelligentRanker ranker = new KendraIntelligentRanker(invalidClient); + + // Otherwise valid search request: + SearchRequest originalRequest = new SearchRequest() + .source(new SearchSourceBuilder() + .query(new MatchAllQueryBuilder())); + KendraIntelligentRankingProperties properties = + new KendraIntelligentRankingProperties(List.of("body"), List.of("title"), 10); + + ResultTransformerConfiguration configuration = new KendraIntelligentRankingConfiguration(1, properties); + boolean shouldTransform = ranker.shouldTransform(originalRequest, configuration); + assertFalse(shouldTransform); + } + public void testShouldNotTransformIfFromExceedsDocLimit() { KendraIntelligentRanker ranker = new KendraIntelligentRanker(buildMockHttpClient()); SearchRequest originalRequest = new SearchRequest() @@ -157,10 +180,37 @@ public void testTransformEmptyHits() { assertSame(searchHits, transformedHits); } - public void testTransformHits() throws IOException { + public void testDoesNotTransformIfMissingBodyField() throws IOException { SearchRequest originalRequest = new SearchRequest() .source(new SearchSourceBuilder().query(new MatchQueryBuilder("body", "foo"))); + int docLimit = randomIntBetween(1, 20); + KendraIntelligentRankingProperties properties = + new KendraIntelligentRankingProperties(List.of("body"), List.of("title"), docLimit); + ResultTransformerConfiguration configuration = new KendraIntelligentRankingConfiguration(1, properties); + SearchHit[] hitsArray = new SearchHit[]{ + new SearchHit(1, "doc1", Map.of(), Map.of()) + .sourceRef(BytesReference.bytes(JsonXContent.contentBuilder() + .startObject() + .field("_id", "1") + .field("title", "This is the title for document 1") + .endObject())) + }; + SearchHits searchHits = new SearchHits(hitsArray, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); + + AtomicBoolean didTransform = new AtomicBoolean(false); + KendraIntelligentRanker ranker = new KendraIntelligentRanker(buildMockHttpClient(r -> { + didTransform.set(true); + return new RescoreResult(); + })); + ranker.transform(searchHits, originalRequest, configuration); + assertFalse(didTransform.get()); + } + + public void testTransformHits() throws IOException { + SearchRequest originalRequest = new SearchRequest() + .source(new SearchSourceBuilder().query(new MultiMatchQueryBuilder("foo", "body", "title"))); + int docLimit = randomIntBetween(1, 20); KendraIntelligentRankingProperties properties = new KendraIntelligentRankingProperties(List.of("body"), List.of("title"), docLimit); @@ -185,12 +235,12 @@ public void testTransformHits() throws IOException { rescoreRequestRef.set(req); // Return the top N results in reverse order. List resultItems = req.getDocuments().stream() - .map(d -> { - RescoreResultItem item = new RescoreResultItem(); - item.setDocumentId(d.getGroupId()); - item.setScore(randomFloat()); - return item; - }).collect(Collectors.toList()); + .map(d -> { + RescoreResultItem item = new RescoreResultItem(); + item.setDocumentId(d.getGroupId()); + item.setScore(randomFloat()); + return item; + }).collect(Collectors.toList()); Collections.reverse(resultItems); RescoreResult result = new RescoreResult(); result.setResultItems(resultItems); @@ -207,6 +257,11 @@ public void testTransformHits() throws IOException { for (int i = docLimit; i < numHits; i++) { assertEquals("doc" + i, transformedHits.getHits()[i].getId()); } + + SearchRequest bodyOnlyRequest = new SearchRequest() + .source(new SearchSourceBuilder().query(new MatchQueryBuilder("body", "foo"))); + SearchHits bodyOnlyTransformedHits = ranker.transform(searchHits, bodyOnlyRequest, configuration); + assertArrayEquals(transformedHits.getHits(), bodyOnlyTransformedHits.getHits()); } } \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java new file mode 100644 index 0000000..ec14710 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.kendraintelligentranking.client; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSSessionCredentials; +import org.opensearch.common.settings.MockSecureSettings; +import org.opensearch.common.settings.SecureSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsException; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class KendraClientSettingsTests extends OpenSearchTestCase { + + private static final String REGION = "us-west-2"; + private static final String ENDPOINT = "http://localhost"; + private static final String ACCESS_KEY = "my-access-key"; + private static final String SECRET_KEY = "my-secret-key"; + private static final String ASSUMED_ROLE = "assumed-role"; + private static final String SESSION_TOKEN = "session-token"; + + + private static KendraClientSettings buildClientSettings(boolean withAccessKey, boolean withSecretKey, + boolean withSessionToken) throws IOException { + try (MockSecureSettings secureSettings = new MockSecureSettings()) { + if (withAccessKey) { + secureSettings.setString(KendraIntelligentRankerSettings.ACCESS_KEY_SETTING.getKey(), ACCESS_KEY); + } + if (withSecretKey) { + secureSettings.setString(KendraIntelligentRankerSettings.SECRET_KEY_SETTING.getKey(), SECRET_KEY); + } + if (withSessionToken) { + secureSettings.setString(KendraIntelligentRankerSettings.SESSION_TOKEN_SETTING.getKey(), SESSION_TOKEN); + } + Settings settings = Settings.builder() + .put(KendraIntelligentRankerSettings.SERVICE_REGION_SETTING.getKey(), REGION) + .put(KendraIntelligentRankerSettings.SERVICE_ENDPOINT_SETTING.getKey(), ENDPOINT) + .put(KendraIntelligentRankerSettings.ASSUME_ROLE_ARN_SETTING.getKey(), ASSUMED_ROLE) + .setSecureSettings(secureSettings) + .build(); + + return KendraClientSettings.getClientSettings(settings); + } + } + + public void testWithBasicCredentials() throws IOException { + KendraClientSettings clientSettings = buildClientSettings(true, true, false); + + AWSCredentials credentials = clientSettings.getCredentials(); + assertEquals(ACCESS_KEY, credentials.getAWSAccessKeyId()); + assertEquals(SECRET_KEY, credentials.getAWSSecretKey()); + assertFalse(credentials instanceof AWSSessionCredentials); + assertEquals(REGION, clientSettings.getServiceRegion()); + assertEquals(ENDPOINT, clientSettings.getServiceEndpoint()); + assertEquals(ASSUMED_ROLE, clientSettings.getAssumeRoleArn()); + } + + public void testWithSessionCredentials() throws IOException { + KendraClientSettings clientSettings = buildClientSettings(true, true, true); + + AWSCredentials credentials = clientSettings.getCredentials(); + assertEquals(ACCESS_KEY, credentials.getAWSAccessKeyId()); + assertEquals(SECRET_KEY, credentials.getAWSSecretKey()); + assertTrue(credentials instanceof AWSSessionCredentials); + AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials; + assertEquals(SESSION_TOKEN, sessionCredentials.getSessionToken()); + assertEquals(REGION, clientSettings.getServiceRegion()); + assertEquals(ENDPOINT, clientSettings.getServiceEndpoint()); + assertEquals(ASSUMED_ROLE, clientSettings.getAssumeRoleArn()); + } + + public void testWithoutCredentials() throws IOException { + KendraClientSettings clientSettings = buildClientSettings(false, false, false); + + assertNull(clientSettings.getCredentials()); + assertEquals(REGION, clientSettings.getServiceRegion()); + assertEquals(ENDPOINT, clientSettings.getServiceEndpoint()); + assertEquals(ASSUMED_ROLE, clientSettings.getAssumeRoleArn()); + } + + public void testWithoutAccessKey() { + expectThrows(SettingsException.class, () -> buildClientSettings(false, true, false)); + expectThrows(SettingsException.class, () -> buildClientSettings(false, true, true)); + } + + public void testWithoutSecretKey() { + expectThrows(SettingsException.class, () -> buildClientSettings(true, false, false)); + expectThrows(SettingsException.class, () -> buildClientSettings(true, false, true)); + } + + public void testWithSessionTokenButNoCredentials() { + expectThrows(SettingsException.class, () -> buildClientSettings(false, false, true)); + } +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java new file mode 100644 index 0000000..3123b31 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.kendraintelligentranking.client; + +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.http.IdleConnectionReaper; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.URI; + +public class KendraHttpClientTests extends OpenSearchTestCase { + + public void testCreateClient() throws Exception { + + BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials("accessKey", "secretKey"); + KendraClientSettings settings = new KendraClientSettings(basicAWSCredentials, + "http://localhost", + "us-west-2", + "12345678", + "myAwesomeRole" + ); + + try (KendraHttpClient client = new KendraHttpClient(settings)) { + assertEquals(new URI("http://localhost/rescore-execution-plans/12345678/rescore"), client.buildRescoreURI()); + } + IdleConnectionReaper.shutdown(); + } + +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java new file mode 100644 index 0000000..e459f02 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.kendraintelligentranking.client; + +import org.mockito.Mockito; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreRequest; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreResult; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.function.Function; + +public class KendraIntelligentClientTests extends OpenSearchTestCase { + protected static KendraHttpClient buildMockHttpClient(Function mockRescoreImpl) { + KendraHttpClient kendraHttpClient = Mockito.mock(KendraHttpClient.class); + Mockito.when(kendraHttpClient.isValid()).thenReturn(true); + Mockito.doAnswer(invocation -> { + RescoreRequest rescoreRequest = invocation.getArgument(0); + return mockRescoreImpl.apply(rescoreRequest); + }).when(kendraHttpClient).rescore(Mockito.any(RescoreRequest.class)); + return kendraHttpClient; + } + + protected static KendraHttpClient buildMockHttpClient() { + return buildMockHttpClient(r -> new RescoreResult()); + } + +} diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java new file mode 100644 index 0000000..f79c99e --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.kendraintelligentranking.client; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.DefaultRequest; +import com.amazonaws.Request; +import com.amazonaws.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +public class SimpleAwsErrorHandlerTests extends OpenSearchTestCase { + + private static final int STATUS_CODE = HttpStatus.SC_PAYMENT_REQUIRED; + private static final String STATUS_TEXT = "Payment required"; + private static final String SERVICE_NAME = "my-service"; + private static final String ERROR_MESSAGE = "This is the error message"; + + public void testBehavior() throws Exception { + SimpleAwsErrorHandler errorHandler = new SimpleAwsErrorHandler(); + assertFalse(errorHandler.needsConnectionLeftOpen()); + Request request = new DefaultRequest<>(SERVICE_NAME); + HttpResponse httpResponse = new HttpResponse(request, null, null); + httpResponse.setContent(new ByteArrayInputStream(ERROR_MESSAGE.getBytes(StandardCharsets.UTF_8))); + httpResponse.setStatusCode(STATUS_CODE); + httpResponse.setStatusText(STATUS_TEXT); + + AmazonServiceException ase = errorHandler.handle(httpResponse); + + assertEquals(ERROR_MESSAGE, ase.getErrorMessage()); + assertEquals(STATUS_CODE, ase.getStatusCode()); + assertEquals(STATUS_TEXT, ase.getErrorCode()); + assertEquals(SERVICE_NAME, ase.getServiceName()); + } +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java new file mode 100644 index 0000000..48a307c --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration; + +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.relevance.configuration.Constants; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; + +public class KendraIntelligentRankingConfigurationTests extends OpenSearchTestCase { + public void testParseWithNullParserAndContext() throws IOException { + expectThrows(NullPointerException.class, () -> KendraIntelligentRankingConfiguration.parse(null)); + } + + public void testSerializeToXContentRoundtrip() throws IOException { + KendraIntelligentRankingConfiguration expected = getKendraIntelligentRankingConfiguration(); + + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference serialized = XContentHelper.toXContent(expected, xContentType, true); + + XContentParser parser = createParser(xContentType.xContent(), serialized); + + KendraIntelligentRankingConfiguration deserialized = + KendraIntelligentRankingConfiguration.parse(parser); + assertEquals(expected, deserialized); + } + + public void testSerializeToStreamRoundtrip() throws IOException { + KendraIntelligentRankingConfiguration expected = getKendraIntelligentRankingConfiguration(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + expected.writeTo(bytesStreamOutput); + + KendraIntelligentRankingConfiguration deserialized = + new KendraIntelligentRankingConfiguration(bytesStreamOutput.bytes().streamInput()); + assertEquals(expected, deserialized); + } + + private static KendraIntelligentRankingConfiguration getKendraIntelligentRankingConfiguration() { + int order = randomInt(10) + 1; + int docLimit = randomInt( Integer.MAX_VALUE - 25) + 25; + KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties properties = + new KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties(List.of("body1"), + List.of("title1"), docLimit); + return new KendraIntelligentRankingConfiguration(order, properties); + } + + public void testReadFromSettings() throws IOException { + int order = randomInt(10); + int docLimit = randomInt(100) + 25; + String bodyField = "body1"; + String titleField = "title1"; + Settings settings = Settings.builder() + .put(Constants.ORDER, order) + .put("properties.doc_limit", docLimit) + .put("properties.body_field", bodyField) + .putList("properties.title_field", titleField) + .build(); + + KendraIntelligentRankingConfiguration expected = new KendraIntelligentRankingConfiguration(order, + new KendraIntelligentRankingConfiguration.KendraIntelligentRankingProperties(List.of(bodyField), + List.of(titleField), docLimit)); + + KendraIntelligentRankingConfiguration actual = new KendraIntelligentRankingConfiguration(settings); + + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java new file mode 100644 index 0000000..43fe08e --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.kendraintelligentranking.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class KendraIntelligentRankingExceptionTests extends OpenSearchTestCase { + + public void testSerializationRoundtrip() throws IOException { + KendraIntelligentRankingException expected = new KendraIntelligentRankingException("This is an error message"); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + expected.writeTo(bytesStreamOutput); + + KendraIntelligentRankingException deserialized = + new KendraIntelligentRankingException(bytesStreamOutput.bytes().streamInput()); + assertEquals(expected.getMessage(), deserialized.getMessage()); + } +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java new file mode 100644 index 0000000..4112f18 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto; + + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +public class DocumentTests extends OpenSearchTestCase { + + + public static final String JSON_DOCUMENT = "{" + + "\"Id\":\"docId\"," + + "\"GroupId\":\"groupIp\"," + + "\"TokenizedTitle\":[\"this\",\"is\",\"a\",\"title\"]," + + "\"TokenizedBody\":[\"here\",\"lies\",\"the\",\"body\"]," + + "\"OriginalScore\":2.71828" + + "}"; + public static final Document TEST_DOCUMENT = new Document("docId", + "groupIp", + List.of("this", "is", "a", "title"), + List.of("here", "lies", "the", "body"), + 2.71828f); + + public void testSerialization() throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + String jsonForm = objectMapper.writeValueAsString(TEST_DOCUMENT); + assertEquals(JSON_DOCUMENT, jsonForm); + } + + public void testDeserialization() throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + Document doc = objectMapper.readValue(JSON_DOCUMENT, Document.class); + assertEquals(TEST_DOCUMENT, doc); + } + +} \ No newline at end of file diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java new file mode 100644 index 0000000..4a42e3d --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java @@ -0,0 +1,137 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.kendraintelligentranking.pipeline; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.OpenSearchParseException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.env.TestEnvironment; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraIntelligentClientTests; + +import java.util.*; + + +public class KendraRankingResponseProcessorTests extends KendraIntelligentClientTests { + private static final String TYPE = "kendra_ranking"; + private Settings settings = buildEnvSettings(Settings.EMPTY); + private Environment env = TestEnvironment.newEnvironment(settings); + + private KendraClientSettings clientSettings = KendraClientSettings.getClientSettings(env.settings()); + + private SearchRequest createRequest() { + QueryBuilder query = new MatchQueryBuilder("body", "value"); + SearchSourceBuilder source = new SearchSourceBuilder().query(query); + return new SearchRequest().source(source); + } + + private SearchResponse createResponse(int size) { + SearchHit[] hits = new SearchHit[size]; + for (int i = 0; i < size; i++) { + Map searchHitFields = new HashMap<>(); + searchHitFields.put("field", new DocumentField("value" + i, Collections.emptyList())); + searchHitFields.put("body", new DocumentField("body" + i, Collections.emptyList())); + hits[i] = new SearchHit(i, "doc " + i, searchHitFields, Collections.emptyMap()); + hits[i].sourceRef(new BytesArray("{ \"field "+ "\" : \"value"+ i + "\" ,\"body"+ "\" : \"body"+ i + "\" }")); + hits[i].score(i); + } + SearchHits searchHits = new SearchHits(hits, new TotalHits(size * 2L, TotalHits.Relation.EQUAL_TO), size); + SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + return new SearchResponse(searchResponseSections, null, 1, 1, 0, 10, null, null); + } + + public void testFactory() throws Exception { + + KendraRankingResponseProcessor.Factory factory = new KendraRankingResponseProcessor.Factory( this.clientSettings); + + //test create without title field, expect exceptions + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + Collections.emptyMap(), + null + )); + + //test create with all fields + List titleField= new ArrayList<>(); + titleField.add("field"); + Map configuration = new HashMap<>(); + configuration.put("title_field","field"); + configuration.put("body_field","body"); + configuration.put("doc_limit","500"); + KendraRankingResponseProcessor processorWithAllFields = factory.create(Collections.emptyMap(),"tmp0","testingAllFields", false, configuration,null); + assertEquals(TYPE, processorWithAllFields.getType()); + assertEquals("tmp0", processorWithAllFields.getTag()); + assertEquals("testingAllFields", processorWithAllFields.getDescription()); + + //test create with required field + Map shortConfiguration = new HashMap<>(); + shortConfiguration.put("body_field","body"); + KendraRankingResponseProcessor processorWithOneFields = factory.create(Collections.emptyMap(),"tmp1","testingBodyField", false, shortConfiguration, null); + assertEquals(TYPE, processorWithOneFields.getType()); + assertEquals("tmp1", processorWithOneFields.getTag()); + assertEquals("testingBodyField", processorWithOneFields.getDescription()); + + //test create with null doc_limit field + Map nullDocLimitConfiguration = new HashMap<>(); + nullDocLimitConfiguration.put("body_field","body"); + nullDocLimitConfiguration.put("doc_limit",null); + KendraRankingResponseProcessor processorWithNullDocLimit = factory.create(Collections.emptyMap(),"tmp2","testingNullDocLimit", false, nullDocLimitConfiguration, null ); + assertEquals(TYPE, processorWithNullDocLimit.getType()); + assertEquals("tmp2", processorWithNullDocLimit.getTag()); + assertEquals("testingNullDocLimit", processorWithNullDocLimit.getDescription()); + + //test create with null title field + Map nullTitleConfiguration = new HashMap<>(); + nullTitleConfiguration.put("body_field","body"); + nullTitleConfiguration.put("title_field",null); + KendraRankingResponseProcessor processorWithNullTitleField = factory.create(Collections.emptyMap(),"tmp3","testingNullTitleField", false, nullTitleConfiguration, null); + assertEquals(TYPE, processorWithNullTitleField.getType()); + assertEquals("tmp3", processorWithNullTitleField.getTag()); + assertEquals("testingNullTitleField", processorWithNullTitleField.getDescription()); + + } + public void testRankingResponse() throws Exception { + KendraHttpClient kendraClient = buildMockHttpClient(); + List titleField = new ArrayList<>(); + titleField.add("field"); + List bodyField = new ArrayList<>(); + bodyField.add("body"); + + //test response with titleField, bodyField and docLimit + KendraRankingResponseProcessor processorWtOptionalConfig = new KendraRankingResponseProcessor(null,null,false, titleField,bodyField,500,kendraClient); + int size = 5; + SearchResponse reRankedResponse0 = processorWtOptionalConfig.processResponse(createRequest(),createResponse(size)); + assertEquals(size,reRankedResponse0.getHits().getHits().length); + + //test response with null doc limit + KendraRankingResponseProcessor processorWtTwoConfig = new KendraRankingResponseProcessor(null,null,false, titleField,bodyField,null,kendraClient); + SearchResponse reRankedResponse1 = processorWtTwoConfig.processResponse(createRequest(),createResponse(size)); + assertEquals(size,reRankedResponse1.getHits().getHits().length); + + //test response with null doc limit and null title field + KendraRankingResponseProcessor processorWtOneConfig = new KendraRankingResponseProcessor(null,null,false,null,bodyField,null,kendraClient); + SearchResponse reRankedResponse2 = processorWtOneConfig.processResponse(createRequest(),createResponse(size)); + assertEquals(size,reRankedResponse2.getHits().getHits().length); + + } +} diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java diff --git a/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java b/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java new file mode 100644 index 0000000..ae37662 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + + +public class AmazonKendraIntelligentRankingClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + + public AmazonKendraIntelligentRankingClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 0000000..af5e5fe --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,17 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^opensearch-amazon-kendra-intelligent-ranking-\d+.\d+.\d+.\d+\n$/ + + - do: + indices.create: + index: test + + - do: + search: + index: test + body: { } diff --git a/amazon-personalize-ranking/build.gradle b/amazon-personalize-ranking/build.gradle new file mode 100644 index 0000000..527d985 --- /dev/null +++ b/amazon-personalize-ranking/build.gradle @@ -0,0 +1,129 @@ +import org.opensearch.gradle.test.RestIntegTestTask + +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'jacoco' + +group = 'org.opensearch' + +def pluginName = 'amazon-personalize-ranking' +def pluginDescription = 'Rerank search results using Amazon Personalize' +def projectPath = 'org.opensearch' +def pathToPlugin = 'search.relevance' +def pluginClassName = 'AmazonPersonalizeRankingPlugin' + + +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}.0" + version "${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') +} + +java { + targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 +} + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + +buildscript { + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + implementation 'org.apache.httpcomponents:httpclient:4.5.14' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-personalizeruntime:1.12.300' + implementation 'commons-logging:commons-logging:1.2' +} + + +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' + } +} + + +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +testClusters.integTest { + testDistribution = "ARCHIVE" + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) +} + +run { + useCluster testClusters.integTest +} + +sourceSets { + main { + resources { + srcDirs = ["config"] + includes = ["**/*.yml"] + } + } +} + + +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java new file mode 100644 index 0000000..343fae5 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchPipelinePlugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchResponseProcessor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParametersExtBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class AmazonPersonalizeRankingPlugin extends Plugin implements SearchPlugin, SearchPipelinePlugin { + + private PersonalizeClientSettings personalizeClientSettings; + + @Override + public List> getSettings() { + // Add settings for other transformers here + return new ArrayList<>(PersonalizeClientSettings.getAllSettings()); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + this.personalizeClientSettings = PersonalizeClientSettings.getClientSettings(environment.settings()); + + return Collections.emptyList(); + } + + @Override + public List> getSearchExts() { + return List.of( + new SearchPlugin.SearchExtSpec<>(PersonalizeRequestParametersExtBuilder.NAME, + PersonalizeRequestParametersExtBuilder::new, + PersonalizeRequestParametersExtBuilder::parse)); + } + + @Override + public Map> getResponseProcessors(Parameters parameters) { + return Map.of(PersonalizeRankingResponseProcessor.TYPE, new PersonalizeRankingResponseProcessor.Factory(this.personalizeClientSettings)); + } +} \ No newline at end of file diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java new file mode 100644 index 0000000..670915c --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking; + +import com.amazonaws.auth.AWSCredentialsProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.ingest.ConfigurationUtils; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.search.pipeline.AbstractProcessor; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchResponseProcessor; +import org.opensearch.search.profile.SearchProfileShardResults; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeCredentialsProviderFactory; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameterUtil; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRanker; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRankerFactory; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.ValidationUtil; + +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; + +/** + * This is a {@link SearchResponseProcessor} that applies Personalized intelligent ranking + */ +public class PersonalizeRankingResponseProcessor extends AbstractProcessor implements SearchResponseProcessor { + + private static final Logger logger = LogManager.getLogger(PersonalizeRankingResponseProcessor.class); + + public static final String TYPE = "personalized_search_ranking"; + private final String tag; + private final String description; + private final PersonalizeClient personalizeClient; + private final PersonalizeIntelligentRankerConfiguration rankerConfig; + + /** + * Constructor for Personalize ranking response processor + * + * @param tag processor tag + * @param description processor description + * @param ignoreFailure processor ignoreFailure config + * @param rankerConfig personalize ranker config + * @param client personalize client + */ + public PersonalizeRankingResponseProcessor(String tag, + String description, + boolean ignoreFailure, + PersonalizeIntelligentRankerConfiguration rankerConfig, + PersonalizeClient client) { + super(tag, description, ignoreFailure); + this.tag = tag; + this.description = description; + this.rankerConfig = rankerConfig; + this.personalizeClient = client; + } + + /** + * Transform the response hits by re ranking results using Personalize + * + * @param request Search request + * @param response Search response that needs to be transformed + * @return Transformed search response using personalized re ranking + * @throws Exception Throws exception for any error while processing response + */ + @Override + public SearchResponse processResponse(SearchRequest request, SearchResponse response) throws Exception { + SearchHits hits = response.getHits(); + + if (hits.getHits().length == 0) { + logger.info("TotalHits = 0. Returning search response without applying Personalize transform"); + return response; + } + logger.info("Personalizing search results."); + PersonalizeRequestParameters personalizeRequestParameters = + PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + PersonalizedRankerFactory rankerFactory = new PersonalizedRankerFactory(); + PersonalizedRanker ranker = rankerFactory.getPersonalizedRanker(rankerConfig, personalizeClient); + long startTime = System.nanoTime(); + SearchHits personalizedHits = ranker.rerank(hits, personalizeRequestParameters); + long personalizeTimeTookMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); + + final SearchResponseSections transformedSearchResponseSections = new InternalSearchResponse(personalizedHits, + (InternalAggregations) response.getAggregations(), response.getSuggest(), + new SearchProfileShardResults(response.getProfileResults()), response.isTimedOut(), + response.isTerminatedEarly(), response.getNumReducePhases()); + + final SearchResponse transformedResponse = new SearchResponse(transformedSearchResponseSections, response.getScrollId(), + response.getTotalShards(), response.getSuccessfulShards(), + response.getSkippedShards(), response.getTook().getMillis() + personalizeTimeTookMs, response.getShardFailures(), + response.getClusters()); + + logger.info("Personalize ranking processor took " + personalizeTimeTookMs + " ms"); + + return transformedResponse; + } + + /** + * Get the type of the processor. + */ + @Override + public String getType() { + return TYPE; + } + + /** + * Get the tag of a processor. + */ + @Override + public String getTag() { + return tag; + } + + /** + * Gets the description of a processor. + */ + @Override + public String getDescription() { + return description; + } + + public static final class Factory implements Processor.Factory { + + private static final String CAMPAIGN_ARN_CONFIG_NAME = "campaign_arn"; + private static final String ITEM_ID_FIELD_CONFIG_NAME = "item_id_field"; + private static final String IAM_ROLE_ARN_CONFIG_NAME = "iam_role_arn"; + private static final String RECIPE_CONFIG_NAME = "recipe"; + private static final String REGION_CONFIG_NAME = "aws_region"; + private static final String WEIGHT_CONFIG_NAME = "weight"; + PersonalizeClientSettings personalizeClientSettings; + private final BiFunction clientBuilder; + + Factory(PersonalizeClientSettings settings, BiFunction clientBuilder) { + this.personalizeClientSettings = settings; + this.clientBuilder = clientBuilder; + } + + public Factory(PersonalizeClientSettings settings) { + this(settings, PersonalizeClient::new); + } + + @Override + public PersonalizeRankingResponseProcessor create(Map> processorFactories, String tag, String description, boolean ignoreFailure, Map config, PipelineContext pipelineContext) { + String personalizeCampaign = ConfigurationUtils.readStringProperty(TYPE, tag, config, CAMPAIGN_ARN_CONFIG_NAME); + String iamRoleArn = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, IAM_ROLE_ARN_CONFIG_NAME); + String recipe = ConfigurationUtils.readStringProperty(TYPE, tag, config, RECIPE_CONFIG_NAME); + String itemIdField = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, ITEM_ID_FIELD_CONFIG_NAME); + String awsRegion = ConfigurationUtils.readStringProperty(TYPE, tag, config, REGION_CONFIG_NAME); + double weight = ConfigurationUtils.readDoubleProperty(TYPE, tag, config, WEIGHT_CONFIG_NAME); + + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, awsRegion, weight); + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, tag); + + final PersonalizeClient personalizeClient; + switch (pipelineContext.getPipelineSource()) { + case SEARCH_REQUEST: + throw new IllegalStateException(TYPE + " processor may not be instantiated as part of a search request. Create a named search pipeline instead."); + case UPDATE_PIPELINE: + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(personalizeClientSettings, iamRoleArn, awsRegion); + personalizeClient = clientBuilder.apply(credentialsProvider, awsRegion); + break; + case VALIDATE_PIPELINE: + default: + personalizeClient = null; // Do not instantiate client on validation + } + return new PersonalizeRankingResponseProcessor(tag, description, ignoreFailure, rankerConfig, personalizeClient); + } + } + + PersonalizeClient getPersonalizeClient() { + // Visible for testing + return personalizeClient; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClient.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClient.java new file mode 100644 index 0000000..738acf2 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClient.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.personalizeruntime.AmazonPersonalizeRuntime; +import com.amazonaws.services.personalizeruntime.AmazonPersonalizeRuntimeClientBuilder; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingRequest; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingResult; + +import java.io.Closeable; +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * Amazon Personalize client implementation for getting personalized ranking + */ +public class PersonalizeClient implements Closeable { + private final AmazonPersonalizeRuntime personalizeRuntime; + private static final String USER_AGENT_PREFIX = "PersonalizeOpenSearchPlugin"; + + /** + * Constructor for Amazon Personalize client + * @param credentialsProvider Credentials to be used for accessing Amazon Personalize + * @param awsRegion AWS region where Amazon Personalize campaign is hosted + */ + public PersonalizeClient(AWSCredentialsProvider credentialsProvider, String awsRegion) { + ClientConfiguration clientConfiguration = AccessController.doPrivileged( + (PrivilegedAction) () -> new ClientConfiguration() + .withUserAgentPrefix(USER_AGENT_PREFIX)); + personalizeRuntime = AccessController.doPrivileged( + (PrivilegedAction) () -> AmazonPersonalizeRuntimeClientBuilder.standard() + .withCredentials(credentialsProvider) + .withRegion(awsRegion) + .withClientConfiguration(clientConfiguration) + .build()); + } + + /** + * Get Personalize runtime client + * @return Personalize runtime client + */ + public AmazonPersonalizeRuntime getPersonalizeRuntime() { + return personalizeRuntime; + } + + /** + * Get Personalized ranking using Personalized runtime client + * @param request Get personalized ranking request + * @return Personalized ranking results + */ + public GetPersonalizedRankingResult getPersonalizedRanking(GetPersonalizedRankingRequest request) { + GetPersonalizedRankingResult result; + try { + result = AccessController.doPrivileged( + (PrivilegedAction) () -> personalizeRuntime.getPersonalizedRanking(request)); + } catch (AmazonServiceException ex) { + throw ex; + } + return result; + } + + @Override + public void close() throws IOException { + if (personalizeRuntime != null) { + personalizeRuntime.shutdown(); + } + } +} \ No newline at end of file diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java new file mode 100644 index 0000000..1a019b5 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.settings.SecureString; +import org.opensearch.common.settings.SecureSetting; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsException; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Container for personalize client settings such as AWS credentials + */ +public final class PersonalizeClientSettings { + + private static final Logger logger = LogManager.getLogger(PersonalizeClientSettings.class); + + /** + * The access key (ie login id) for connecting to Personalize. + */ + public static final Setting ACCESS_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.access_key", null); + + /** + * The secret key (ie password) for connecting to Personalize. + */ + public static final Setting SECRET_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.secret_key", null); + + /** + * The session token for connecting to Personalize. + */ + public static final Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.session_token", null); + + private final AWSCredentials credentials; + + protected PersonalizeClientSettings(AWSCredentials credentials) { + this.credentials = credentials; + } + + public static Collection> getAllSettings() { + return Arrays.asList( + ACCESS_KEY_SETTING, + SECRET_KEY_SETTING, + SESSION_TOKEN_SETTING + ); + } + + public AWSCredentials getCredentials() { + return credentials; + } + + /** + * Load AWS credentials from open search keystore if available + * @param settings Open search settings + * @return AWS credentials + */ + static AWSCredentials loadCredentials(Settings settings) { + try (SecureString key = ACCESS_KEY_SETTING.get(settings); + SecureString secret = SECRET_KEY_SETTING.get(settings); + SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) { + if (key.length() == 0 && secret.length() == 0) { + if (sessionToken.length() > 0) { + throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not", + SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + logger.info("Using either environment variables, system properties or instance profile credentials"); + return null; + } else if (key.length() == 0 || secret.length() == 0) { + throw new SettingsException("One of settings [{}] and [{}] is not set.", + ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } else { + final AWSCredentials credentials; + if (sessionToken.length() == 0) { + logger.info("Using basic key/secret credentials"); + credentials = new BasicAWSCredentials(key.toString(), secret.toString()); + } else { + logger.info("Using basic session credentials"); + credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString()); + } + return credentials; + } + } + } + + /** + * Get Personalize client settings + * @param settings Open search settings + * @return Personalize client settings instance with AWS credentials + */ + public static PersonalizeClientSettings getClientSettings(Settings settings) { + final AWSCredentials credentials = loadCredentials(settings); + return new PersonalizeClientSettings(credentials); + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java new file mode 100644 index 0000000..e0e2895 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * Factory implementation for getting Personalize credentials + */ +public final class PersonalizeCredentialsProviderFactory { + private static final Logger logger = LogManager.getLogger(PersonalizeCredentialsProviderFactory.class); + private static final String ASSUME_ROLE_SESSION_NAME = "OpenSearchPersonalizeIntelligentRankingPluginSession"; + + private PersonalizeCredentialsProviderFactory() { + } + + /** + * Get AWS credentials provider either from static credentials from open search keystore or + * using DefaultAWSCredentialsProviderChain. + * @param clientSettings Personalize client settings + * @return AWS credentials provider for accessing Personalize + */ + static AWSCredentialsProvider getCredentialsProvider(PersonalizeClientSettings clientSettings) { + final AWSCredentialsProvider credentialsProvider; + final AWSCredentials credentials = clientSettings.getCredentials(); + if (credentials == null) { + logger.info("Credentials not present in open search keystore. Using DefaultAWSCredentialsProviderChain for credentials."); + credentialsProvider = AccessController.doPrivileged( + (PrivilegedAction) () -> DefaultAWSCredentialsProviderChain.getInstance()); + } else { + logger.info("Using credentials provided in open search keystore"); + credentialsProvider = AccessController.doPrivileged( + (PrivilegedAction) () -> new AWSStaticCredentialsProvider(credentials)); + } + return credentialsProvider; + } + + /** + * Get AWS credentials provider by assuming IAM role if provided or else + * use static credentials or DefaultAWSCredentialsProviderChain. + * @param clientSettings Personalize client settings + * @param personalizeIAMRole IAM role configuration for accessing Personalize + * @param awsRegion AWS region + * @return AWS credentials provider for accessing Amazon Personalize + */ + public static AWSCredentialsProvider getCredentialsProvider(PersonalizeClientSettings clientSettings, + String personalizeIAMRole, + String awsRegion) { + + final AWSCredentialsProvider credentialsProvider; + AWSCredentialsProvider baseCredentialsProvider = getCredentialsProvider(clientSettings); + + if (personalizeIAMRole != null && !personalizeIAMRole.isBlank()) { + logger.info("Using IAM Role provided to access Personalize."); + // If IAM role ARN was provided in config, then use auto-refreshed role credentials. + credentialsProvider = AccessController.doPrivileged( + (PrivilegedAction) () -> { + AWSSecurityTokenService awsSecurityTokenService = AWSSecurityTokenServiceClientBuilder.standard() + .withCredentials(baseCredentialsProvider) + .withRegion(awsRegion) + .build(); + + return new STSAssumeRoleSessionCredentialsProvider.Builder(personalizeIAMRole, ASSUME_ROLE_SESSION_NAME) + .withStsClient(awsSecurityTokenService) + .build(); + }); + } else { + logger.info("IAM Role for accessing Personalize is not provided."); + credentialsProvider = baseCredentialsProvider; + } + return credentialsProvider; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java new file mode 100644 index 0000000..40105f9 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java @@ -0,0 +1,17 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration; + +/** + * Constants for Amazon Perosnalize response processor + */ +public class Constants { + public static final String AMAZON_PERSONALIZED_RANKING_RECIPE_NAME = "aws-personalized-ranking"; + public static final String AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME = "aws-personalized-ranking-v2"; +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java new file mode 100644 index 0000000..35685de --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration; + +/** + * A container for holding Personalize ranker configuration + */ +public class PersonalizeIntelligentRankerConfiguration { + private final String personalizeCampaign; + private final String iamRoleArn; + private final String recipe; + private final String itemIdField; + private final String region; + private final double weight; + + /** + * + * @param personalizeCampaign Personalize campaign + * @param iamRoleArn IAM Role ARN for accessing Personalize campaign + * @param recipe Personalize recipe associated with campaign + * @param itemIdField Item ID field to pick up item id for Personalize input + * @param region AWS region + * @param weight Configurable coefficient to control Personalization of search results + */ + public PersonalizeIntelligentRankerConfiguration(String personalizeCampaign, + String iamRoleArn, + String recipe, + String itemIdField, + String region, + double weight) { + this.personalizeCampaign = personalizeCampaign; + this.iamRoleArn = iamRoleArn; + this.recipe = recipe; + this.itemIdField = itemIdField; + this.region = region; + this.weight = weight; + } + + /** + * Get PErsonalize campaign + * @return Personalize campaign + */ + public String getPersonalizeCampaign() { + return personalizeCampaign; + } + + /** + * Get recipe + * @return Recipe associated with Personalize campaign + */ + public String getRecipe() { + return recipe; + } + + /** + * Get Item ID field + * @return Item ID field + */ + public String getItemIdField() { + return itemIdField; + } + + /** + * Get AWS region + * @return AWS region + */ + public String getRegion() { + return region; + } + + /** + * + * @return weight value + */ + public double getWeight() { + return weight; + } + + /** + * Get IAM role ARN for Personalize campaign + * @return IAM role for accessing Personalize campaign + */ + public String getIamRoleArn() { + return iamRoleArn; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java new file mode 100644 index 0000000..bb36927 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.search.SearchExtBuilder; + +import java.util.List; +import java.util.stream.Collectors; + +public class PersonalizeRequestParameterUtil { + + public static PersonalizeRequestParameters getPersonalizeRequestParameters(SearchRequest searchRequest) { + PersonalizeRequestParametersExtBuilder personalizeRequestParameterExtBuilder = null; + if (searchRequest.source() != null && searchRequest.source().ext() != null && !searchRequest.source().ext().isEmpty()) { + List extBuilders = searchRequest.source().ext().stream() + .filter(extBuilder -> PersonalizeRequestParametersExtBuilder.NAME.equals(extBuilder.getWriteableName())) + .collect(Collectors.toList()); + + if (!extBuilders.isEmpty()) { + personalizeRequestParameterExtBuilder = (PersonalizeRequestParametersExtBuilder) extBuilders.get(0); + } + } + PersonalizeRequestParameters personalizeRequestParameters = null; + if (personalizeRequestParameterExtBuilder != null) { + personalizeRequestParameters = personalizeRequestParameterExtBuilder.getRequestParameters(); + } + return personalizeRequestParameters; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java new file mode 100644 index 0000000..cb7a102 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.ParseField; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class PersonalizeRequestParameters implements Writeable, ToXContentObject { + + static final String PERSONALIZE_REQUEST_PARAMETERS = "personalize_request_parameters"; + private static final String USER_ID_PARAMETER = "user_id"; + private static final String CONTEXT_PARAMETER = "context"; + + private static final ObjectParser PARSER; + private static final ParseField USER_ID = new ParseField(USER_ID_PARAMETER); + private static final ParseField CONTEXT = new ParseField(CONTEXT_PARAMETER); + + static { + PARSER = new ObjectParser<>(PERSONALIZE_REQUEST_PARAMETERS, PersonalizeRequestParameters::new); + PARSER.declareString(PersonalizeRequestParameters::setUserId, USER_ID); + PARSER.declareObject(PersonalizeRequestParameters::setContext,(XContentParser p, Void c) -> { + try { + return p.map(); + } catch (IOException e) { + throw new IllegalArgumentException("Error parsing Personalize context from request parameters", e); + } + }, CONTEXT); + } + + private String userId; + + private Map context; + + public PersonalizeRequestParameters() {} + + public PersonalizeRequestParameters(String userId, Map context) { + this.userId = userId; + this.context = context; + } + + public PersonalizeRequestParameters(StreamInput input) throws IOException { + this.userId = input.readString(); + this.context = input.readMap(); + } + + public String getUserId() { + return userId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public Map getContext() { + return context; + } + + public void setContext(Map context) { + this.context = context; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(this.userId); + out.writeMap(this.context); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(USER_ID.getPreferredName(), this.userId); + return builder.field(CONTEXT.getPreferredName(), this.context); + } + + public static PersonalizeRequestParameters parse(XContentParser parser) throws IOException { + PersonalizeRequestParameters requestParameters = PARSER.parse(parser, null); + return requestParameters; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PersonalizeRequestParameters config = (PersonalizeRequestParameters) o; + + if (!userId.equals(config.userId)) return false; + if (context.size() != config.getContext().size()) return false; + return userId.equals(config.userId) && context.equals(config.getContext()); + } + + @Override + public int hashCode() { + return Objects.hash(userId, context); + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java new file mode 100644 index 0000000..a7b6e8c --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchExtBuilder; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters.PERSONALIZE_REQUEST_PARAMETERS; + +public class PersonalizeRequestParametersExtBuilder extends SearchExtBuilder { + private static final Logger logger = LogManager.getLogger(PersonalizeRequestParametersExtBuilder.class); + public static final String NAME = PERSONALIZE_REQUEST_PARAMETERS; + private PersonalizeRequestParameters requestParameters; + + public PersonalizeRequestParametersExtBuilder() {} + + public PersonalizeRequestParametersExtBuilder(StreamInput input) throws IOException { + requestParameters = new PersonalizeRequestParameters(input); + } + + public PersonalizeRequestParameters getRequestParameters() { + return requestParameters; + } + + public void setRequestParameters(PersonalizeRequestParameters requestParameters) { + this.requestParameters = requestParameters; + } + + @Override + public int hashCode() { + return Objects.hash(this.getClass(), this.requestParameters); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof PersonalizeRequestParametersExtBuilder)) { + return false; + } + PersonalizeRequestParametersExtBuilder o = (PersonalizeRequestParametersExtBuilder) obj; + return this.requestParameters.equals(o.requestParameters); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + requestParameters.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.value(requestParameters); + } + + public static PersonalizeRequestParametersExtBuilder parse(XContentParser parser) throws IOException{ + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + PersonalizeRequestParameters requestParameters = PersonalizeRequestParameters.parse(parser); + extBuilder.setRequestParameters(requestParameters); + return extBuilder; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java new file mode 100644 index 0000000..8470f9f --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker; + +import org.opensearch.search.SearchHits; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; + +public interface PersonalizedRanker { + + /** + * Re rank search hits + * @param hits Search hits to re rank + * @param requestParameters Request parameters for Personalize present in search request + * @return Re ranked search hits + */ + SearchHits rerank(SearchHits hits, PersonalizeRequestParameters requestParameters); +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java new file mode 100644 index 0000000..ead8dd8 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl.AmazonPersonalizedRankerImpl; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; + +/** + * Factory for creating Personalize ranker instance based on Personalize ranker configuration + */ +public class PersonalizedRankerFactory { + private static final Logger logger = LogManager.getLogger(PersonalizedRankerFactory.class); + + /** + * Create an instance of Personalize ranker based on ranker configuration + * @param config Personalize ranker configuration + * @param client Personalize client + * @return Personalize ranker instance + */ + public PersonalizedRanker getPersonalizedRanker(PersonalizeIntelligentRankerConfiguration config, PersonalizeClient client){ + PersonalizedRanker ranker = null; + String recipeInConfig = config.getRecipe(); + if (recipeInConfig.equals(AMAZON_PERSONALIZED_RANKING_RECIPE_NAME) + || recipeInConfig.equals(AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME)) { + ranker = new AmazonPersonalizedRankerImpl(config, client); + } else { + logger.error("Personalize recipe provided in configuration is not supported for re ranking search results"); + //TODO : throw user error exception + } + return ranker; + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java new file mode 100644 index 0000000..142dceb --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java @@ -0,0 +1,171 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingRequest; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingResult; +import com.amazonaws.services.personalizeruntime.model.PredictedItem; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ingest.ConfigurationUtils; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRanker; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.ValidationUtil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Personalize Re Ranker implementation using Amazon Personalized Ranking recipe + */ +public class AmazonPersonalizedRankerImpl implements PersonalizedRanker { + private static final Logger logger = LogManager.getLogger(AmazonPersonalizedRankerImpl.class); + private final PersonalizeIntelligentRankerConfiguration rankerConfig; + private final PersonalizeClient personalizeClient; + + public AmazonPersonalizedRankerImpl(PersonalizeIntelligentRankerConfiguration config, + PersonalizeClient client) { + this.rankerConfig = config; + this.personalizeClient = client; + } + + /** + * Re rank search hits using Personalize campaign that uses Personalized Ranking recipe + * @param hits search hits returned by open search + * @param requestParameters request parameters for Personalize present in search request + * @return search hots re ranked using Amazon Personalize + */ + @Override + public SearchHits rerank(SearchHits hits, PersonalizeRequestParameters requestParameters) { + try { + validatePersonalizeRequestParams(requestParameters); + List originalHits = Arrays.asList(hits.getHits()); + // Do not make Personalize call if weight is zero which implies Personalization is turned off. + if (rankerConfig.getWeight() == 0) { + logger.info("Not applying Personalized ranking. Given value for weight configuration: {}", rankerConfig.getWeight()); + return hits; + } + String itemIdfield = rankerConfig.getItemIdField(); + List documentIdsToRank; + // If item field is not specified in the configuration then use default _id field. + if (itemIdfield != null && !itemIdfield.isBlank()) { + documentIdsToRank = originalHits.stream() + .filter(h -> h.getSourceAsMap().get(itemIdfield) != null) + .map(h -> h.getSourceAsMap().get(itemIdfield).toString()) + .collect(Collectors.toList()); + } else { + documentIdsToRank = originalHits.stream() + .filter(h -> h.getId() != null) + .map(h -> h.getId()) + .collect(Collectors.toList()); + } + if (documentIdsToRank.size() == 0) { + throw ConfigurationUtils.newConfigurationException(PersonalizeRankingResponseProcessor.TYPE, "", "item_id_field", + "no item ids found to apply Personalized reranking. Please check configured value for item_id_field"); + } + logger.info("Document Ids to re-rank with Personalize: {}", Arrays.toString(documentIdsToRank.toArray())); + String userId = requestParameters.getUserId(); + Map context = requestParameters.getContext() != null ? + requestParameters.getContext().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> (String)e.getValue())) + : null; + logger.info("User ID from personalize request parameters - User ID: {}", userId); + if (context != null && !context.isEmpty()) { + logger.info("Personalize context provided in the search request"); + } + + GetPersonalizedRankingRequest personalizeRequest = new GetPersonalizedRankingRequest() + .withCampaignArn(rankerConfig.getPersonalizeCampaign()) + .withInputList(documentIdsToRank) + .withContext(context) + .withUserId(userId); + GetPersonalizedRankingResult result = personalizeClient.getPersonalizedRanking(personalizeRequest); + + SearchHits personalizedHits = combineScores(hits, result); + return personalizedHits; + } catch (AmazonServiceException e) { + logger.error("Exception while calling personalize campaign: {}", e.getMessage()); + int statusCode = e.getStatusCode(); + if (ValidationUtil.is4xxError(statusCode)) { + throw new IllegalArgumentException(e); + } + throw e; + } + catch (Exception ex) { + logger.error("Failed to re rank with Personalize.", ex); + throw ex; + } + } + + //Combine open search hits and personalize campaign response + private SearchHits combineScores(SearchHits originalHits, GetPersonalizedRankingResult personalizedRankingResult) { + List personalziedRanking = personalizedRankingResult.getPersonalizedRanking(); + List personalizedRankedItemsList = new LinkedList<>(); + for (PredictedItem item : personalziedRanking) { + personalizedRankedItemsList.add(item.getItemId()); + } + int totalHits = originalHits.getHits().length; + List rerankedHits = new ArrayList<>(totalHits); + float maxScore = 0f; + double weight = rankerConfig.getWeight(); + for (int i = 0 ; i < totalHits ; i++) { + String openSearchItemId; + SearchHit hit = originalHits.getAt(i); + String itemIdField = rankerConfig.getItemIdField(); + if (itemIdField != null && !(itemIdField.isBlank())) { + openSearchItemId = hit.getSourceAsMap().get(rankerConfig.getItemIdField()).toString(); + } else { + openSearchItemId = hit.getId(); + } + int openSearchRank = i + 1; + int personalizedRank = personalizedRankedItemsList.indexOf(openSearchItemId) + 1; + float combinedScore = (float) (((1- weight) / (Math.log(openSearchRank + 1) / Math.log(2))) + + ((weight) / (Math.log(personalizedRank + 1) / Math.log(2)))); + maxScore = Math.max(maxScore, combinedScore); + hit.score(combinedScore); + rerankedHits.add(hit); + } + rerankedHits.sort(Comparator.comparing(SearchHit::getScore).reversed()); + return new SearchHits(rerankedHits.toArray(new SearchHit[0]), originalHits.getTotalHits(), maxScore); + } + + /** + * Validate Personalize configuration for calling Personalize service + * @param requestParameters Request parameters for Personalize present in search request + */ + private void validatePersonalizeRequestParams(PersonalizeRequestParameters requestParameters) { + if (requestParameters == null || requestParameters.getUserId() == null || requestParameters.getUserId().isBlank()) { + throw ConfigurationUtils.newConfigurationException(PersonalizeRankingResponseProcessor.TYPE, "", "user_id", + "required Personalize request parameter is missing"); + } + if (requestParameters.getContext() != null) { + try { + requestParameters.getContext().entrySet().stream().forEach(e -> isValidPersonalizeContext(e)); + } catch (IllegalArgumentException iae) { + throw ConfigurationUtils.newConfigurationException(PersonalizeRankingResponseProcessor.TYPE, "", "context", iae.getMessage()); + } + } + } + + private void isValidPersonalizeContext(Map.Entry contextEntry) throws IllegalArgumentException { + if (!(contextEntry.getValue() instanceof String)) { + throw new IllegalArgumentException("Personalize context value is not of type String. Invalid context value: " + contextEntry.getValue()); + } + } +} diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java new file mode 100644 index 0000000..c6c92f7 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils; +import com.amazonaws.arn.Arn; +import org.opensearch.ingest.ConfigurationUtils; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; + +import java.util.Arrays; +import java.util.Set; +import java.util.HashSet; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; + +public class ValidationUtil { + private static Set SUPPORTED_PERSONALIZE_RECIPES = new HashSet<>(Arrays.asList( + AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, + AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME + )); + + /** + * Validate Personalize configuration for calling Personalize service. + * Throws OpenSearchParseException type exception if validation fails. + * @param config Personalize intelligent ranker configuration + * @param processorType Name of search pipeline processor + * @param processorTag Name of processor tag + */ + public static void validatePersonalizeIntelligentRankerConfiguration (PersonalizeIntelligentRankerConfiguration config, + String processorType, + String processorTag + ) { + // Validate weight value + if (config.getWeight() < 0.0 || config.getWeight() > 1.0) { + throw ConfigurationUtils.newConfigurationException(processorType, processorTag, "weight", "invalid value for weight"); + } + // Validate Personalize campaign ARN + if(!isValidCampaignOrRoleArn(config.getPersonalizeCampaign(), "personalize")) { + throw ConfigurationUtils.newConfigurationException(processorType, processorTag, "campaign_arn", "invalid format for Personalize campaign arn"); + } + // Validate IAM Role Arn for Personalize access + String iamRoleArn = config.getIamRoleArn(); + if(!(iamRoleArn == null || iamRoleArn.isBlank()) && !isValidCampaignOrRoleArn(iamRoleArn, "iam")) { + throw ConfigurationUtils.newConfigurationException(processorType, processorTag, "iam_role_arn", "invalid format for Personalize iam role arn"); + } + // Validate Personalize recipe + if(!SUPPORTED_PERSONALIZE_RECIPES.contains(config.getRecipe())) { + throw ConfigurationUtils.newConfigurationException(processorType, processorTag, "recipe", "not supported recipe provided"); + } + } + + private static boolean isValidCampaignOrRoleArn(String arn, String expectedService) { + try { + Arn arnObj = Arn.fromString(arn); + return arnObj.getService().equals(expectedService); + } catch (IllegalArgumentException iae) { + return false; + } + } + + public static boolean is4xxError(int statusCode){ + if (statusCode >= 400 && statusCode < 500) { + return true; + } + return false; + } +} diff --git a/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy b/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000..b16dfe0 --- /dev/null +++ b/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + permission java.net.SocketPermission "*", "connect,resolve"; + permission java.lang.RuntimePermission "getClassLoader"; +}; diff --git a/src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java similarity index 66% rename from src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java index fc54c71..f72f947 100644 --- a/src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java @@ -7,21 +7,22 @@ */ package org.opensearch.search.relevance; -import org.apache.http.util.EntityUtils; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; -public class SearchRelevancePluginIT extends OpenSearchRestTestCase { +public class AmazonPersonalizeRankingPluginIT extends OpenSearchRestTestCase { - public void testPluginInstalled() throws IOException { + public void testPluginInstalled() throws IOException, ParseException { Response response = client().performRequest(new Request("GET", "/_cat/plugins")); String body = EntityUtils.toString(response.getEntity()); logger.info("response body: {}", body); assertNotNull(body); - assertTrue(body.contains("search-processor")); + assertTrue(body.contains("amazon-personalize-ranking")); } } diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java new file mode 100644 index 0000000..f83bbcb --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java @@ -0,0 +1,435 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking; + +import com.amazonaws.http.IdleConnectionReaper; +import org.apache.lucene.search.TotalHits; +import org.opensearch.OpenSearchParseException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.search.SearchResponseSections; +import org.opensearch.action.search.ShardSearchFailure; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.env.TestEnvironment; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeRuntimeTestUtil; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.SearchTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor.TYPE; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; + +public class PersonalizeRankingResponseProcessorTests extends OpenSearchTestCase { + + private static final Processor.PipelineContext UPDATE_CONTEXT = new Processor.PipelineContext(Processor.PipelineSource.UPDATE_PIPELINE); + private static final Processor.PipelineContext VALIDATE_CONTEXT = new Processor.PipelineContext(Processor.PipelineSource.VALIDATE_PIPELINE); + private final Settings settings = buildEnvSettings(Settings.EMPTY); + private final Environment env = TestEnvironment.newEnvironment(settings); + private static final String PERSONALIZE_CAMPAIGN = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + private static final String IAM_ROLE_ARN = "arn:aws:iam::000000000000:role/test"; + private static final String ITEM_ID_FIELD = "ITEM_ID"; + private static final String REGION = "us-west-2"; + private static final double WEIGHT = 1.0; + private static final int NUM_HITS = 10; + + private final PersonalizeClientSettings clientSettings = PersonalizeClientSettings.getClientSettings(env.settings()); + + public void testCreateFactoryThrowsExceptionWithEmptyConfig() { + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings); + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + Collections.emptyMap(), + UPDATE_CONTEXT + )); + IdleConnectionReaper.shutdown(); + } + + public void testFactoryValidations() { + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings); + // Test config without campaign + Map configuration = new HashMap<>(); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + configuration, + VALIDATE_CONTEXT + )); + configuration.clear(); + + // Test config without recipe + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + configuration, + VALIDATE_CONTEXT + )); + configuration.clear(); + + // Test config without region + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + configuration, + VALIDATE_CONTEXT + )); + configuration.clear(); + + // Test config without weight + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + configuration, + VALIDATE_CONTEXT + )); + configuration.clear(); + + // Test configuration with invalid weight value + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", "invalid"); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + expectThrows(OpenSearchParseException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + configuration, + VALIDATE_CONTEXT + )); + configuration.clear(); + + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + + // Test that we don't create client on validation + configuration.putAll(buildPersonalizeResponseProcessorConfig()); + PersonalizeRankingResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, false, configuration, VALIDATE_CONTEXT); + assertNull(processor.getPersonalizeClient()); + + // Test that we fail on valid configuration in search request context + expectThrows(IllegalStateException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + buildPersonalizeResponseProcessorConfig(), + new Processor.PipelineContext(Processor.PipelineSource.SEARCH_REQUEST))); + } + + public void testCreateFactoryWithAllPersonalizeConfig() throws Exception { + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings); + + Map configuration = buildPersonalizeResponseProcessorConfig(); + + PersonalizeRankingResponseProcessor personalizeResponseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + assertEquals(TYPE, personalizeResponseProcessor.getType()); + assertEquals("testTag", personalizeResponseProcessor.getTag()); + assertEquals("testingAllFields", personalizeResponseProcessor.getDescription()); + IdleConnectionReaper.shutdown(); + } + + public void testProcessorWithNoHits() throws Exception { + PersonalizeClient mockClient = mock(PersonalizeClient.class); + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> mockClient); + + Map configuration = buildPersonalizeResponseProcessorConfig(); + + PersonalizeRankingResponseProcessor personalizeResponseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + SearchRequest searchRequest = new SearchRequest(); + SearchHits hits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0.0f); + SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, false, null, 0); + SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 1, new ShardSearchFailure[0], null); + + SearchResponse response = personalizeResponseProcessor.processResponse(searchRequest, searchResponse); + assertEquals(hits.getTotalHits().value, response.getHits().getTotalHits().value); + IdleConnectionReaper.shutdown(); + } + + public void testProcessorWithPersonalizeContext() throws Exception { + PersonalizeClient mockClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> mockClient); + + Map configuration = buildPersonalizeResponseProcessorConfig(); + PersonalizeRankingResponseProcessor personalizeResponseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + Map personalizeContext = new HashMap<>(); + personalizeContext.put("contextKey2", "contextValue2"); + + SearchResponse personalizedResponse = + createPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .filter(h -> h.getSourceAsMap().get(ITEM_ID_FIELD) != null) + .map(h -> h.getSourceAsMap().get(ITEM_ID_FIELD).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testProcessorWithHitsWithInvalidPersonalizeContext() throws Exception { + PersonalizeClient mockClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient();; + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> mockClient); + + Map configuration = buildPersonalizeResponseProcessorConfig(); + PersonalizeRankingResponseProcessor personalizeResponseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + Map personalizeContext = new HashMap<>(); + personalizeContext.put("contextKey2", 5); + + expectThrows(OpenSearchParseException.class, () -> + createPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, NUM_HITS)); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingResponse() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemField = "ITEM_ID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .filter(h -> h.getSourceAsMap().get(itemField) != null) + .map(h -> h.getSourceAsMap().get(itemField).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2Response() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemField = "ITEM_ID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .filter(h -> h.getSourceAsMap().get(itemField) != null) + .map(h -> h.getSourceAsMap().get(itemField).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2ResponseWithInvalidItemIdFieldName() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemFieldInvalid = "ITEM_ID_NOT_VALID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + configuration.put("item_id_field", itemFieldInvalid); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + expectThrows(OpenSearchParseException.class, () -> + createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS)); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2ResponseWithDefaultItemIdField() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemIdFieldEmpty = ""; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("item_id_field", itemIdFieldEmpty); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .map(SearchHit::getId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingResponseWithInvalidItemIdFieldName() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemFieldInvalid = "ITEM_ID_NOT_VALID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("item_id_field", itemFieldInvalid); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + expectThrows(OpenSearchParseException.class, () -> + createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS)); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingResponseWithDefaultItemIdField() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemIdFieldEmpty = ""; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("item_id_field", itemIdFieldEmpty); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .map(SearchHit::getId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + private SearchResponse createPersonalizedRankingProcessorResponse(PersonalizeRankingResponseProcessor responseProcessor, + Map personalizeContext, + int numHits) throws Exception { + + PersonalizeRequestParameters personalizeRequestParams = new PersonalizeRequestParameters("user_1", personalizeContext); + SearchRequest request = SearchTestUtil.createSearchRequestWithPersonalizeRequest(personalizeRequestParams); + + SearchHits searchHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numHits); + SearchResponseSections searchResponseSections = new SearchResponseSections(searchHits, null, null, false, false, null, 0); + SearchResponse searchResponse = new SearchResponse(searchResponseSections, null, 1, 1, 0, 1, new ShardSearchFailure[0], null); + + SearchResponse personalizedResponse = responseProcessor.processResponse(request, searchResponse); + + return personalizedResponse; + } + + private Map buildPersonalizeResponseProcessorConfig() { + Map configuration = new HashMap<>(); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + return configuration; + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java new file mode 100644 index 0000000..bf1def8 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSSessionCredentials; +import org.opensearch.common.settings.SecureSetting; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.SettingsException; +import org.opensearch.core.common.settings.SecureString; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil.ACCESS_KEY; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil.SECRET_KEY; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil.SESSION_TOKEN; + +public class PersonalizeClientSettingsTests extends OpenSearchTestCase { + + public void testWithBasicCredentials() throws IOException { + PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, false); + AWSCredentials credentials = clientSettings.getCredentials(); + assertEquals(ACCESS_KEY, credentials.getAWSAccessKeyId()); + assertEquals(SECRET_KEY, credentials.getAWSSecretKey()); + assertFalse(credentials instanceof AWSSessionCredentials); + } + + public void testWithGetAllSetting() throws IOException { + PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + assertEquals(clientSettings.getAllSettings().size(), 3); + Setting ACCESS_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.access_key", null); + Setting SECRET_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.secret_key", null); + Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.session_token", null); + assertEquals(ACCESS_KEY_SETTING, clientSettings.getAllSettings().toArray()[0]); + assertEquals(SECRET_KEY_SETTING, clientSettings.getAllSettings().toArray()[1]); + assertEquals(SESSION_TOKEN_SETTING, clientSettings.getAllSettings().toArray()[2]); + } + + public void testWithSessionCredentials() throws IOException { + PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + AWSCredentials credentials = clientSettings.getCredentials(); + assertEquals(ACCESS_KEY, credentials.getAWSAccessKeyId()); + assertEquals(SECRET_KEY, credentials.getAWSSecretKey()); + assertTrue(credentials instanceof AWSSessionCredentials); + AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials; + assertEquals(SESSION_TOKEN, sessionCredentials.getSessionToken()); + } + + public void testWithoutCredentials() throws IOException { + PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(false, false, false); + assertNull(clientSettings.getCredentials()); + } + + public void testWithoutAccessKey() { + expectThrows(SettingsException.class, () -> PersonalizeClientSettingsTestUtil.buildClientSettings(false, true, false)); + expectThrows(SettingsException.class, () -> PersonalizeClientSettingsTestUtil.buildClientSettings(false, true, true)); + } + + public void testWithoutSecretKey() { + expectThrows(SettingsException.class, () -> PersonalizeClientSettingsTestUtil.buildClientSettings(true, false, false)); + expectThrows(SettingsException.class, () -> PersonalizeClientSettingsTestUtil.buildClientSettings(true, false, true)); + } + + public void testWithSessionTokenButNoCredentials() { + expectThrows(SettingsException.class, () -> PersonalizeClientSettingsTestUtil.buildClientSettings(false, false, true)); + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java new file mode 100644 index 0000000..57c2ead --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingRequest; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingResult; +import org.mockito.Mockito; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeRuntimeTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.mockito.ArgumentMatchers.any; + +public class PersonalizeClientTests extends OpenSearchTestCase { + + public void testCreateClient() throws IOException { + AWSCredentials credentials = new BasicSessionCredentials("accessKey", "secretKey", "sessionToken"); + AWSCredentialsProvider credentialsProvider = new AWSStaticCredentialsProvider(credentials); + String region = "us-west-2"; + try (PersonalizeClient client = new PersonalizeClient(credentialsProvider,region)) { + assertTrue(client.getPersonalizeRuntime() != null); + } + } + + public void testGetPersonalizedRanking() { + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + GetPersonalizedRankingRequest request = PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingRequest(); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + GetPersonalizedRankingResult result = client.getPersonalizedRanking(request); + assertEquals(result.getRecommendationId(), "sampleRecommendationId"); + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java new file mode 100644 index 0000000..af72a46 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.client; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; +import com.amazonaws.http.IdleConnectionReaper; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class PersonalizeCredentialsProviderFactoryTests extends OpenSearchTestCase { + + public void testGetStaticCredentialsProviderWithoutIAMRole() throws IOException { + PersonalizeClientSettings settings = + PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(settings); + assertEquals(credentialsProvider.getClass(), AWSStaticCredentialsProvider.class); + } + + public void testGetDefaultCredentialsProviderWithoutIAMRole() throws IOException { + PersonalizeClientSettings settings = + PersonalizeClientSettingsTestUtil.buildClientSettings(false, false, false); + + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(settings); + assertEquals(credentialsProvider.getClass(), DefaultAWSCredentialsProviderChain.class); + } + + public void testGetCredentialsProviderWithIAMRole() throws IOException { + PersonalizeClientSettings settings = + PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + + String iamRoleArn = "test-iam-role-arn"; + String awsRegion = "us-west-2"; + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(settings, iamRoleArn, awsRegion); + assertEquals(credentialsProvider.getClass(), STSAssumeRoleSessionCredentialsProvider.class); + IdleConnectionReaper.shutdown(); + } + + public void testGetStaticCredentialsProviderWithEmptyIAMRole() throws IOException { + PersonalizeClientSettings settings = + PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + + String iamRoleArn = ""; + String awsRegion = "us-west-2"; + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(settings, iamRoleArn, awsRegion); + assertEquals(credentialsProvider.getClass(), AWSStaticCredentialsProvider.class); + } + + public void testGetDefaultCredentialsProviderWithEmptyIAMRole() throws IOException { + PersonalizeClientSettings settings = + PersonalizeClientSettingsTestUtil.buildClientSettings(false, false, false); + + String iamRoleArn = ""; + String awsRegion = "us-west-2"; + AWSCredentialsProvider credentialsProvider = PersonalizeCredentialsProviderFactory.getCredentialsProvider(settings, iamRoleArn, awsRegion); + assertEquals(credentialsProvider.getClass(), DefaultAWSCredentialsProviderChain.class); + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java new file mode 100644 index 0000000..b54a2e4 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration; + +import org.opensearch.test.OpenSearchTestCase; + +public class PersonalizeIntelligentRankerConfigurationTests extends OpenSearchTestCase { + + public void createConfigurationTest() { + String personalizeCampaign = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + String iamRoleArn = "sampleRoleArn"; + String recipe = "sample-personalize-recipe"; + String itemIdField = "ITEM_ID"; + String region = "us-west-2"; + double weight = 0.25; + + PersonalizeIntelligentRankerConfiguration config = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + + assertEquals(config.getPersonalizeCampaign(), personalizeCampaign); + assertEquals(config.getIamRoleArn(), iamRoleArn); + assertEquals(config.getRecipe(), recipe); + assertEquals(config.getItemIdField(), itemIdField); + assertEquals(config.getRegion(), region); + assertEquals(config.getWeight(), weight, 0.0); + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java new file mode 100644 index 0000000..b88f601 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.ranker; + +import org.mockito.Mockito; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRanker; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRankerFactory; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl.AmazonPersonalizedRankerImpl; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; + +public class PersonalizeRankerFactoryTests extends OpenSearchTestCase { + + private String personalizeCampaign = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + private String iamRoleArn = "sampleRoleArn"; + private String itemIdField = "ITEM_ID"; + private String region = "us-west-2"; + private double weight = 0.25; + + public void testGetPersonalizeRankerForPersonalizedRankingRecipe() { + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + + PersonalizedRankerFactory factory = new PersonalizedRankerFactory(); + PersonalizedRanker ranker = factory.getPersonalizedRanker(rankerConfig, client); + assertEquals(ranker.getClass(), AmazonPersonalizedRankerImpl.class); + } + + public void testGetPersonalizeRankerForUnknownRecipe() { + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, "sample-recipe", itemIdField, region, weight); + + PersonalizedRankerFactory factory = new PersonalizedRankerFactory(); + PersonalizedRanker ranker = factory.getPersonalizedRanker(rankerConfig, client); + assertNull(ranker); + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java new file mode 100644 index 0000000..1eec5cc --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java @@ -0,0 +1,343 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.ranker.impl; + +import com.amazonaws.AmazonServiceException; +import org.junit.Assert; +import org.mockito.Mockito; +import org.opensearch.OpenSearchParseException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl.AmazonPersonalizedRankerImpl; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeRuntimeTestUtil; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.SearchTestUtil; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.any; + +public class AmazonPersonalizeRankerImplTests extends OpenSearchTestCase { + + private String personalizeCampaign = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + private String iamRoleArn = "sampleRoleArn"; + private String recipe = "sample-personalize-recipe"; + private String itemIdField = "ITEM_ID"; + private String region = "us-west-2"; + private double weight = 0.25; + private int numOfHits = 10; + + public void testReRank() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + assertEquals(responseHits.getHits().length, transformedHits.getHits().length); + } + + public void testReRankWithoutItemIdFieldInConfig() throws IOException { + String blankItemIdField = ""; + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, blankItemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + assertEquals(responseHits.getHits().length, transformedHits.getHits().length); + } + + public void testReRankWithRequestParameterContext() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Map context = new HashMap<>(); + context.put("contextKey", "contextValue"); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + requestParameters.setContext(context); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + assertEquals(responseHits.getHits().length, transformedHits.getHits().length); + } + + public void testReRankWithInvalidRequestParameterContext() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Map context = new HashMap<>(); + context.put("contextKey", 2); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + requestParameters.setContext(context); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + expectThrows(OpenSearchParseException.class, () -> + ranker.rerank(responseHits, requestParameters)); + } + + public void testReRankWithNoUserId() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Map context = new HashMap<>(); + context.put("contextKey", "contextValue"); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setContext(context); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + expectThrows(OpenSearchParseException.class, () -> + ranker.rerank(responseHits, requestParameters)); + } + + public void testReRankWithEmptyItemIdField() throws IOException { + String itemIdEmpty = ""; + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdEmpty, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + assertEquals(responseHits.getHits().length, transformedHits.getHits().length); + } + + public void testReRankWithNullItemIdField() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, null, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult()); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + assertEquals(responseHits.getHits().length, transformedHits.getHits().length); + } + + public void testReRankWithWeightAsZero() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, 0); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + String itemIdfield = rankerConfig.getItemIdField(); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getSourceAsMap().get(itemIdfield) != null) + .map(h -> h.getSourceAsMap().get(itemIdfield).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 0); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + } + + public void testReRankWithWeightAsOne() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, 1); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + String itemIdfield = rankerConfig.getItemIdField(); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getSourceAsMap().get(itemIdfield) != null) + .map(h -> h.getSourceAsMap().get(itemIdfield).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + } + + public void testReRankWithWeightAsNeitherZeroOrOne() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + String itemIdfield = rankerConfig.getItemIdField(); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getSourceAsMap().get(itemIdfield) != null) + .map(h -> h.getSourceAsMap().get(itemIdfield).toString()) + .collect(Collectors.toList()); + + ArrayList rerankedDocumentIdsWhenWeightIsOne = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 1); + ArrayList rerankedDocumentIdsWhenWeightIsZero = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 0); + + assertNotEquals(rerankedDocumentIdsWhenWeightIsOne, rerankedDocumentIds); + assertNotEquals(rerankedDocumentIdsWhenWeightIsZero, rerankedDocumentIds); + } + + public void testReRankWithWeightAsZeroWithNullItemIdField() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, "", region, 0); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getId() != null) + .map(h -> h.getId()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 0); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + } + + + public void testReRankWithWeightAsOneWithNullItemIdField() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, "", region, 1); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getId() != null) + .map(h -> h.getId()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + } + + public void testReRankWithWeightAsNeitherZeroOrOneWithNullItemIdField() throws IOException { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, "", region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenReturn(PersonalizeRuntimeTestUtil.buildGetPersonalizedRankingResult(numOfHits)); + + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + SearchHits transformedHits = ranker.rerank(responseHits, requestParameters); + + List originalHits = Arrays.asList(transformedHits.getHits()); + List rerankedDocumentIds; + + rerankedDocumentIds = originalHits.stream() + .filter(h -> h.getId() != null) + .map(h -> h.getId()) + .collect(Collectors.toList()); + + ArrayList rerankedDocumentIdsWhenWeightIsOne = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 1); + ArrayList rerankedDocumentIdsWhenWeightIsZero = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numOfHits, 0); + + assertNotEquals(rerankedDocumentIdsWhenWeightIsOne, rerankedDocumentIds); + assertNotEquals(rerankedDocumentIdsWhenWeightIsZero, rerankedDocumentIds); + } + + public void testReRankWithaccessDeniedExceptionWithStatusCode400() throws IOException { + + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenThrow(buildErrorWithStatusCode(400)); + + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Assert.assertThrows(IllegalArgumentException.class, () -> ranker.rerank(responseHits, requestParameters)); + } + + public void testReRankWithaccessDeniedExceptionWithStatusCode500() throws IOException { + + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenThrow(buildErrorWithStatusCode(500)); + + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Assert.assertThrows(AmazonServiceException.class, () -> ranker.rerank(responseHits, requestParameters)); + } + + + private AmazonServiceException buildErrorWithStatusCode(int statusCode) { + AmazonServiceException amazonServiceException = new AmazonServiceException("Error"); + amazonServiceException.setStatusCode(statusCode); + return amazonServiceException; + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java new file mode 100644 index 0000000..ea09d79 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PersonalizeRequestParameterUtilTests extends OpenSearchTestCase { + + public void testExtractParameters() { + PersonalizeRequestParameters expected = new PersonalizeRequestParameters("user_1", new HashMap<>()); + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(expected); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(sourceBuilder); + PersonalizeRequestParameters actual = PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + assertEquals(expected, actual); + } + + public void testExtractParametersWithContext() { + Map context = new HashMap<>(); + context.put("contextKey", "contextValue"); + PersonalizeRequestParameters expected = new PersonalizeRequestParameters("user_1", context); + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(expected); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(sourceBuilder); + PersonalizeRequestParameters actual = PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + assertEquals(expected, actual); + } + + public void testPersonalizeRequestParametersEquals() { + Map notExpectedContext = new HashMap<>(); + notExpectedContext.put("contextKey", "contextValue"); + PersonalizeRequestParameters notExpected = new PersonalizeRequestParameters("user_1", notExpectedContext); + + Map expectedContext = new HashMap<>(); + expectedContext.put("contextKey2", "contextValue2"); + PersonalizeRequestParameters expected = new PersonalizeRequestParameters("user_1", expectedContext); + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(expected); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(sourceBuilder); + PersonalizeRequestParameters actual = PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + assertNotEquals(notExpected, actual); + } + + public void testPersonalizeRequestParametersContextMapDifferentSize() { + Map notExpectedContext = new HashMap<>(); + notExpectedContext.put("contextKey", "contextValue"); + PersonalizeRequestParameters notExpected = new PersonalizeRequestParameters("user_1", notExpectedContext); + + Map expectedContext = new HashMap<>(); + expectedContext.put("contextKey2", "contextValue2"); + expectedContext.put("contextKey22", "contextValue22"); + PersonalizeRequestParameters expected = new PersonalizeRequestParameters("user_1", expectedContext); + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(expected); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(sourceBuilder); + PersonalizeRequestParameters actual = PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + assertNotEquals(notExpected, actual); + } + + public void testPersonalizeRequestParametersUserIdDiffers() { + Map notExpectedContext = new HashMap<>(); + notExpectedContext.put("contextKey", "contextValue"); + PersonalizeRequestParameters notExpected = new PersonalizeRequestParameters("user_1", notExpectedContext); + + Map expectedContext = new HashMap<>(); + expectedContext.put("contextKey", "contextValue"); + PersonalizeRequestParameters expected = new PersonalizeRequestParameters("user_2", expectedContext); + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(expected); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + SearchRequest request = new SearchRequest("my_index").source(sourceBuilder); + PersonalizeRequestParameters actual = PersonalizeRequestParameterUtil.getPersonalizeRequestParameters(request); + assertNotEquals(notExpected, actual); + } +} \ No newline at end of file diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java new file mode 100644 index 0000000..fc1c8c9 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter; + +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class PersonalizeRequestParametersExtBuilderTests extends OpenSearchTestCase { + + public void testXContentRoundTrip() throws IOException { + Map context = new HashMap<>(); + context.put("contextKey", "contextValue"); + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters("28", context); + PersonalizeRequestParametersExtBuilder personalizeExtBuilder = new PersonalizeRequestParametersExtBuilder(); + personalizeExtBuilder.setRequestParameters(requestParameters); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference serialized = XContentHelper.toXContent(personalizeExtBuilder, xContentType, true); + + XContentParser parser = createParser(xContentType.xContent(), serialized); + + PersonalizeRequestParametersExtBuilder deserialized = + PersonalizeRequestParametersExtBuilder.parse(parser); + + assertEquals(personalizeExtBuilder, deserialized); + } + + public void testStreamRoundTrip() throws IOException { + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + requestParameters.setContext(new HashMap<>()); + PersonalizeRequestParametersExtBuilder personalizeExtBuilder = new PersonalizeRequestParametersExtBuilder(); + personalizeExtBuilder.setRequestParameters(requestParameters); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + personalizeExtBuilder.writeTo(bytesStreamOutput); + + PersonalizeRequestParametersExtBuilder deserialized = + new PersonalizeRequestParametersExtBuilder(bytesStreamOutput.bytes().streamInput()); + assertEquals(personalizeExtBuilder, deserialized); + } + + +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java new file mode 100644 index 0000000..d5e4052 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils; + +import org.opensearch.common.settings.MockSecureSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; + +import java.io.IOException; + +public class PersonalizeClientSettingsTestUtil { + public static final String ACCESS_KEY = "my-access-key"; + public static final String SECRET_KEY = "my-secret-key"; + public static final String SESSION_TOKEN = "session-token"; + + public static PersonalizeClientSettings buildClientSettings(boolean withAccessKey, boolean withSecretKey, + boolean withSessionToken) throws IOException { + try (MockSecureSettings secureSettings = new MockSecureSettings()) { + if (withAccessKey) { + secureSettings.setString(PersonalizeClientSettings.ACCESS_KEY_SETTING.getKey(), ACCESS_KEY); + } + if (withSecretKey) { + secureSettings.setString(PersonalizeClientSettings.SECRET_KEY_SETTING.getKey(), SECRET_KEY); + } + if (withSessionToken) { + secureSettings.setString(PersonalizeClientSettings.SESSION_TOKEN_SETTING.getKey(), SESSION_TOKEN); + } + Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .build(); + return PersonalizeClientSettings.getClientSettings(settings); + } + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java new file mode 100644 index 0000000..4028c3d --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils; + +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingRequest; +import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingResult; +import com.amazonaws.services.personalizeruntime.model.PredictedItem; +import org.mockito.Mockito; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import static org.mockito.ArgumentMatchers.any; + +public class PersonalizeRuntimeTestUtil { + + public static GetPersonalizedRankingRequest buildGetPersonalizedRankingRequest() { + GetPersonalizedRankingRequest request = new GetPersonalizedRankingRequest() + .withUserId("sampleUserId") + .withInputList(new ArrayList()) + .withCampaignArn("sampleCampaign"); + return request; + } + + public static GetPersonalizedRankingResult buildGetPersonalizedRankingResult() { + List predictedItems = new ArrayList<>(); + GetPersonalizedRankingResult result = new GetPersonalizedRankingResult() + .withPersonalizedRanking(predictedItems) + .withRecommendationId("sampleRecommendationId"); + return result; + } + + public static GetPersonalizedRankingResult buildGetPersonalizedRankingResult(int numOfHits) { + List predictedItems = new ArrayList<>(); + for(int i = numOfHits; i >= 1; i--){ + PredictedItem predictedItem = new PredictedItem(). + withScore((double) i/10). + withItemId(String.valueOf(i-1)); + predictedItems.add(predictedItem); + } + GetPersonalizedRankingResult result = new GetPersonalizedRankingResult() + .withPersonalizedRanking(predictedItems) + .withRecommendationId("sampleRecommendationId"); + return result; + } + + public static ArrayList expectedRankedItemIdsForGivenWeight(int numOfHits, int weight){ + ArrayList expectedRankedItemIds = new ArrayList<>(); + if (weight == 0) { + for (int i = 0; i < numOfHits; i++) { + expectedRankedItemIds.add(String.valueOf(i)); + } + } else if (weight == 1){ + for(int i = numOfHits; i >= 1; i--){ + expectedRankedItemIds.add(String.valueOf(i-1)); + } + } + return expectedRankedItemIds; + } + + public static PersonalizeClient buildMockPersonalizeClient() { + return buildMockPersonalizeClient(r -> buildGetPersonalizedRankingResult(10)); + } + + private static PersonalizeClient buildMockPersonalizeClient( + Function mockGetPersonalizedRankingImpl) { + PersonalizeClient personalizeClient = Mockito.mock(PersonalizeClient.class); + Mockito.doAnswer(invocation -> { + GetPersonalizedRankingRequest request = invocation.getArgument(0); + return mockGetPersonalizedRankingImpl.apply(request); + }).when(personalizeClient).getPersonalizedRanking(any(GetPersonalizedRankingRequest.class)); + return personalizeClient; + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java new file mode 100644 index 0000000..ca08126 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils; + +import org.apache.lucene.search.TotalHits; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParametersExtBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class SearchTestUtil { + public static SearchHits getSampleSearchHitsForPersonalize(int numHits) throws IOException { + SearchHit[] hitsArray = new SearchHit[numHits]; + float maxScore = 0.0f; + for (int i = 0; i < numHits; i++) { + XContentBuilder sourceContent = JsonXContent.contentBuilder() + .startObject() + .field("ITEM_ID", String.valueOf(i)) + .field("body", "Body text for document number " + i) + .field("title", "This is the title for document " + i) + .endObject(); + hitsArray[i] = new SearchHit(i, String.valueOf(i), Map.of(), Map.of()); + float score = (float)(numHits-i)/10; + maxScore = Math.max(score, maxScore); + hitsArray[i].score(score); + hitsArray[i].sourceRef(BytesReference.bytes(sourceContent)); + } + SearchHits searchHits = new SearchHits(hitsArray, new TotalHits(numHits, TotalHits.Relation.EQUAL_TO), maxScore); + return searchHits; + } + + public static SearchRequest createSearchRequestWithPersonalizeRequest(PersonalizeRequestParameters personalizeRequestParams) { + PersonalizeRequestParametersExtBuilder extBuilder = new PersonalizeRequestParametersExtBuilder(); + extBuilder.setRequestParameters(personalizeRequestParams); + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource() + .ext(List.of(extBuilder)); + + SearchRequest searchRequest = new SearchRequest().source(sourceBuilder); + return searchRequest; + } +} diff --git a/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java new file mode 100644 index 0000000..1227915 --- /dev/null +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils; + +import com.amazonaws.http.IdleConnectionReaper; +import org.opensearch.OpenSearchParseException; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; +import org.opensearch.test.OpenSearchTestCase; + +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; + +public class ValidationUtilTests extends OpenSearchTestCase { + + private static final String TYPE = "personalize_ranking"; + private static final String TAG = "test_tag"; + private String personalizeCampaign = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + private String iamRoleArn = "arn:aws:iam::000000000000:role/test"; + private String itemIdField = "ITEM_ID"; + private String region = "us-west-2"; + private double weight = 1.0; + + public void testValidRankerConfig () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG); + } + + public void testValidRankerConfigPersonalizedRankingV2 () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME, itemIdField, region, weight); + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG); + } + + public void testInvalidCampaignArn () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration("invalid:campaign/test", iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testEmptyCampaignArn () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration("", iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testNonPersonalizeArnAsCampaignArn () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration("arn:aws:es:us-west-2:000000000000:domain/testmovies", iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testInvalidIamRoleArn () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, "invalid:arn/test", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testNonIamArnAsIamRoleArn () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, "arn:aws:es:us-west-2:000000000000:domain/testmovies", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testEmptyIamRoleArnAllowed () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, "", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG); + } + + public void testInvalidWeightValueGreaterThanRange () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, 3.0); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testInvalidWeightValueLessThanRange () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, -1.0); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } + + public void testNonPersonalizedRankingRecipeConfig () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, "aws-user-personalization", itemIdField, region, -1.0); + expectThrows(OpenSearchParseException.class, () -> + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG)); + } +} diff --git a/src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java b/amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java similarity index 77% rename from src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java rename to amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java index 1f0d7dd..43e0349 100644 --- a/src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java +++ b/amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java @@ -13,9 +13,9 @@ import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; -public class SearchRelevanceClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { +public class AmazonPersonalizeRankingClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { - public SearchRelevanceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + public AmazonPersonalizeRankingClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 0000000..daa9333 --- /dev/null +++ b/amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,17 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^opensearch-amazon-personalize-ranking-\d+.\d+.\d+.\d+\n$/ + + - do: + indices.create: + index: test + + - do: + search: + index: test + body: { } diff --git a/build.gradle b/build.gradle index cc99f6e..3647cd1 100644 --- a/build.gradle +++ b/build.gradle @@ -1,142 +1,8 @@ -import org.opensearch.gradle.test.RestIntegTestTask - -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' -apply plugin: 'opensearch.pluginzip' -apply plugin: 'jacoco' - -group = 'org.opensearch' - -def pluginName = 'search-processor' -def pluginDescription = 'Make Opensearch results more relevant.' -def projectPath = 'org.opensearch' -def pathToPlugin = 'search.relevance' -def pluginClassName = 'SearchRelevancePlugin' - -publishing { - publications { - pluginZip(MavenPublication) { publication -> - pom { - name = pluginName - description = pluginDescription - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - developers { - developer { - name = "OpenSearch" - url = "https://github.com/opensearch-project/search-relevance" - } - } - } - } +ext { + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "3.0.0") + plugin_version = opensearch_version + if (isSnapshot) { + opensearch_version += "-SNAPSHOT" } } -opensearchplugin { - name pluginName - description pluginDescription - classname "${projectPath}.${pathToPlugin}.${pluginClassName}" - licenseFile rootProject.file('LICENSE') - noticeFile rootProject.file('NOTICE') -} - -// This requires an additional Jar not published as part of build-tools -loggerUsageCheck.enabled = false - -// No need to validate pom, as we do not upload to maven/sonatype -validateNebulaPom.enabled = false - -buildscript { - ext { - opensearch_version = "2.4.0" - } - - repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } - } - - dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearch_version}" - } -} - -repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } -} - -dependencies { - implementation 'com.ibm.icu:icu4j:57.2' - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.httpcomponents:httpclient:4.5.13' - implementation 'org.apache.httpcomponents:httpcore:4.4.15' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.0' - implementation 'com.fasterxml.jackson.core:jackson-core:2.14.0' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.14.0' - implementation 'commons-logging:commons-logging:1.2' - implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' - implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' -} - -test { - include '**/*Tests.class' - finalizedBy jacocoTestReport -} - -task integTest(type: RestIntegTestTask) { - description = "Run tests against a cluster" - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath -} -tasks.named("check").configure { dependsOn(integTest) } - -integTest { - // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' - } -} - -testClusters.integTest { - testDistribution = "INTEG_TEST" - - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) -} - -run { - useCluster testClusters.integTest -} - -sourceSets { - main { - resources { - srcDirs = ["config"] - includes = ["**/*.yml"] - } - } -} - - -jacocoTestReport { - reports { - xml.enabled = true - html.enabled = true - } - dependsOn test -} - -// TODO: Enable these checks -dependencyLicenses.enabled = false -thirdPartyAudit.enabled = false -loggerUsageCheck.enabled = false diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 41d9927..a4b76b9 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fc..e1b837a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionSha256Sum=7a00d51fb93147819aab76024feece20b6b84e420694101f276be952e08bef03 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 005bcde..f5feea6 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,12 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +134,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ @@ -205,6 +217,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index 6a68175..9b42019 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,89 +1,94 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS=-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/helpers/personalized_search_ranking_quickstart.sh b/helpers/personalized_search_ranking_quickstart.sh new file mode 100755 index 0000000..c78b3ad --- /dev/null +++ b/helpers/personalized_search_ranking_quickstart.sh @@ -0,0 +1,414 @@ +#!/bin/bash + +set -o errexit +set -o errtrace +set -o pipefail +set -o nounset + +# Some useful constants +readonly DOCKER_IMAGE_TAG="opensearch-with-personalized-search-ranking" +readonly OPENSEARCH_VERSION="2.9.0" + +# +# Set default values for OpenSearch (+Dashboards) image tags and plugin URL. +# We can pass them from outside to override these settings for testing purposes. +# +if [ -z "${OPENSEARCH_IMAGE_TAG:-}" ]; then + OPENSEARCH_IMAGE_TAG="opensearchproject/opensearch:${OPENSEARCH_VERSION}" +fi +if [ -z "${OPENSEARCH_DASHBOARDS_IMAGE_TAG:-}" ]; then + OPENSEARCH_DASHBOARDS_IMAGE_TAG="opensearchproject/opensearch-dashboards:${OPENSEARCH_VERSION}" +fi +if [ -z "${SEARCH_PROCESSOR_PLUGIN_URL:-}" ]; then + SEARCH_PROCESSOR_PLUGIN_URL="https://github.com/opensearch-project/search-processor/releases/download/${OPENSEARCH_VERSION}/opensearch-search-processor-${OPENSEARCH_VERSION}.0.zip" +fi + +function print_help() { + cat << EOF +Usage: $0 [-r ] [--profile ] + [--volume-name ] [--admin-password ] + -r | --region The AWS region for the Personalize Intelligent Ranking + service endpoint. If not specified, will read from the + AWS CLI for the default profile. + --profile The AWS profile to use for credentials. If not set, then + the script will try first to use credentials from the + environment, then from the default AWS profile. + --volume-name Without this option, the OpenSearch container will write + the index to ephemeral container storage, which is lost when + the container is removed. Using this option will map the + named Docker volume to \$OPENSEARCH_ROOT/data, so index data + will persist across executions. If the named volume does not + exist, it will be created. + --admin-password For OpenSearch 2.12 and higher, we no longer use a default + password of "admin" for the admin user. Instead, the value + passed to this parameter will be used as the admin password. + For OpenSearch versions prior to 2.12, this argument will be + ignored with a warning. + + NOTE: If the --profile option is not specified, the script will attempt to read AWS + credentials (access/secret key, optional session token) from environment variables, + and then from the default AWS profile, in order to pass them to the OpenSearch keystore + to be used to connect to the Personalize Intelligent ranking service. + If no credentials are found, the script WILL NOT pass credentials to the OpenSearch + keystore. When running a reranking request, the ranking plugin may rely on + instance profile credentials delivered through the EC2 metadata service, or credentials + from the ECS metadata service. +EOF +} + +# +# Parse and validate arguments +# + +while [ "$#" -gt 0 ]; do + case $1 in + -r | --region ) + shift + AWS_REGION=$1 + shift + ;; + -h | --help ) + print_help + exit 0 + ;; + --profile ) + shift + AWS_PROFILE=$1 + shift + ;; + --volume-name ) + shift + VOLUME_NAME=$1 + shift + ;; + --admin-password ) + shift + OPENSEARCH_INITIAL_ADMIN_PASSWORD="$1" + shift + ;; + esac +done + +# Starting in 2.12.0, security demo configuration script requires an initial admin password +OPENSEARCH_REQUIRED_VERSION="2.12.0" +COMPARE_VERSION=`echo $OPENSEARCH_REQUIRED_VERSION $OPENSEARCH_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` +if [ "$COMPARE_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then + if [ -n "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "WARNING: The --admin-password setting has no effect on OpenSearch ${OPENSEARCH_VERSION}. The admin password will be 'admin'." + fi + OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin" +elif [ -z "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "Starting with OpenSearch 2.12, you must specify the admin password with the --admin-password parameter." + exit 1 +fi + +# +# Determine which credentials and region to use. By the end of this block, all specified +# credentials will be loaded into environment variables (or we fail with an explanatory +# error message). +# +if [ -n "${AWS_PROFILE:-}" ]; then + # Load everything from the specified profile + AWS_ACCESS_KEY_ID=$(aws --profile ${AWS_PROFILE} configure get aws_access_key_id || echo) + AWS_SECRET_ACCESS_KEY=$(aws --profile ${AWS_PROFILE} configure get aws_secret_access_key || echo) + AWS_SESSION_TOKEN=$(aws --profile ${AWS_PROFILE} configure get aws_session_token || echo) + if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + >&2 echo "Unable to load credentials from profile ${AWS_PROFILE}" + exit 1 + elif [ -z "${AWS_SESSION_TOKEN}" ]; then + echo "Using AWS credentials (aws_access_key_id and aws_secret_access_key) from profile ${AWS_PROFILE}" + else + echo "Using AWS credentials (aws_access_key_id, aws_secret_access_key, and aws_session_token) from profile ${AWS_PROFILE}" + fi + if [ -z "${AWS_REGION:-}" ]; then + AWS_REGION=$(aws --profile ${AWS_PROFILE} configure get region || echo) + if [ -n "${AWS_REGION:-}" ]; then + echo "Using AWS region ${AWS_REGION} from profile ${AWS_PROFILE}" + else + >&2 echo "Argument [-r | --region] not specified and unable to infer region from profile ${AWS_PROFILE}" + exit 1 + fi + fi +else + # No profile set + if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then + if [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + echo "No profile set and no credentials in environment. Trying to load default profile credentials." + AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id || echo) + AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key || echo) + AWS_SESSION_TOKEN=$(aws configure get aws_session_token || echo) + if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then + echo "Unable to load credentials from default profile. No credentials will be passed to the OpenSearch keystore." + echo "OpenSearch will use the default credential provider chain to access Personalize, which may rely on EC2 instance" + echo "profile credentials or credentials from ECS metadata service." + elif [ -z "${AWS_SESSION_TOKEN}" ]; then + echo "Using AWS credentials (aws_access_key_id and aws_secret_access_key) from default profile." + else + echo "Using AWS credentials (aws_access_key_id, aws_secret_access_key, and aws_session_token) from default profile." + fi + else + >&2 echo "Environment variable AWS_SECRET_ACCESS_KEY is specified, but AWS_ACCESS_KEY_ID is not." + >&2 echo "Unable to determine which credentials to use." + exit 1 + fi + else + # AWS_ACCCESS_KEY_ID is set + if [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + >&2 echo "Environment variable AWS_ACCESS_KEY_ID is specified, but AWS_SECRET_ACCESS_KEY is not." + >&2 echo "Unable to determine which credentials to use." + exit 1 + else + if [ -n "${AWS_SESSION_TOKEN:-}" ]; then + echo "Using credentials from environment (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN)." + else + echo "Using credentials from environment (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)." + fi + fi + fi + if [ -z "${AWS_REGION:-}" ]; then + AWS_REGION=$(aws configure get region || echo) + if [ -n "${AWS_REGION:-}" ]; then + echo "Using AWS region ${AWS_REGION} from default profile" + else + >&2 echo "Argument [-r | --region] not specified and unable to infer region from default profile" + exit 1 + fi + fi +fi + +echo "Established AWS key id, secret key, session token and region." + +# +# Create a unique directory to hold the Dockerfile and docker-compose.yml files. +# +PLATFORM=$(uname) +if [ "${PLATFORM}" == "Darwin" ]; then + DOCKER_BUILD_DIR=$(mktemp -d opensearch-personalize-intelligent-ranking-docker.XXXX) +else + # Assume GNU mktemp + DOCKER_BUILD_DIR=$(mktemp -d -p . opensearch-personalize-intelligent-ranking-docker.XXXX) +fi + +cd ${DOCKER_BUILD_DIR} + +SUFFIX=$(echo ${DOCKER_BUILD_DIR} | sed s/.*\.//) +echo $SUFFIX + +echo "Running in $(pwd)" + +# +# Construct a Dockerfile that installs the search-processor plugin in the target image +# +cat >Dockerfile <>personalized_search_ranking.credentials <>personalized_search_ranking.credentials <install_credentials.sh <<"EOF" +#!/bin/bash + +for l in $(cat $1); do + KEY=$(echo $l | cut -f1 -d:) + VALUE=$(echo $l | cut -f2 -d:) + echo $VALUE | /usr/share/opensearch/bin/opensearch-keystore add $KEY --stdin +done +EOF + chmod 755 install_credentials.sh + cat >>Dockerfile << EOF + +# Push credentials to keystore +COPY --chown=opensearch:opensearch install_credentials.sh /tmp +COPY --chown=opensearch:opensearch personalized_search_ranking.credentials /tmp +RUN /usr/share/opensearch/bin/opensearch-keystore create +RUN --mount=type=secret,id=credentials,target=/tmp/personalized_search_ranking.credentials,required=true,mode=0444 \ + /tmp/install_credentials.sh /tmp/personalized_search_ranking.credentials +EOF +fi +echo "Opensearch credentials saved" +# +# Build and tag the Docker image with the plugin (and maybe credentials in the keystore) +# +if [ -f personalized_search_ranking.credentials ]; then + DOCKER_BUILDKIT=1 docker build --tag ${DOCKER_IMAGE_TAG} --secret id=credentials,src=personalized_search_ranking.credentials . + rm personalized_search_ranking.credentials +else + docker build --tag ${DOCKER_IMAGE_TAG} . +fi +echo "Docker image built and tagged with credentials" +# +# Make sure we have opensearch-dashboards: +# +docker pull ${OPENSEARCH_DASHBOARDS_IMAGE_TAG} +echo "Docker image pulled" + +if [ -n "${VOLUME_NAME:-}" ]; then + if ! docker volume inspect ${VOLUME_NAME}> /dev/null; then + echo "Creating volume ${VOLUME_NAME}"; + docker volume create ${VOLUME_NAME} + fi + DATA_DIR_BLOCK=" volumes: + - ${VOLUME_NAME}:/usr/share/opensearch/data" + VOLUME_BLOCK="volumes: + ${VOLUME_NAME}: + external: true" +fi +echo "Volume created" + + + +# +# Create a docker-compose.yml file that will launch an OpenSearch node with the image we +# just built and an OpenSearch Dashboards node that points to the OpenSearch node. +# +cat >docker-compose.yml <cleanup_resources.sh <README <" https://localhost:9200/ + +Index some data on OpenSearch by following instructions at +https://opensearch.org/docs/latest/opensearch/index-data/ + + +Connect to OpenSearch Dashboards with a web browser at http://localhost:5601/, +using username admin and password admin. Select "Search Relevance" from the +top-left menu. In the resulting UI, you can submit a query without Personalized +search ranking and one with Personalized search Ranking. + +To configure and setup Personalize search ranking, run a curl command as follows: + +curl -X PUT "https://localhost:9200/_search/pipeline/intelligent_ranking" -u 'admin:' --insecure -H 'Content-Type: application/json' -d' +{ + "description": "A pipeline to apply custom reranking", + "response_processors" : [ + { + "personalized_search_ranking" : { + "campaign_arn" : "", + "item_id_field" : "", + "recipe" : "", + "weight" : "", + "iam_role_arn": "", + "aws_region": "" + } + } + ] +}' + +Interact with the Docker containers using docker-compose from directory + $(pwd) + +Some helpful docker-compose commands: + + docker-compose logs opensearch-node + Outputs latest logs from the OpenSearch server + + docker-compose logs opensearch-dashboard + Outputs latest logs from the OpenSearch Dashboard server + + docker-compose down + Shut down and clean up both containers. + + docker-compose up -d + Bring both containers back up. + +You can clean up all Docker containers and any execution plans created (if +applicable) by running + $(pwd)/cleanup_resources.sh + +The full text of this message is also available at + $(pwd)/README +EOF +cat README diff --git a/helpers/search_processing_kendra_quickstart.sh b/helpers/search_processing_kendra_quickstart.sh index f4a0b92..3c0bba4 100755 --- a/helpers/search_processing_kendra_quickstart.sh +++ b/helpers/search_processing_kendra_quickstart.sh @@ -7,7 +7,7 @@ set -o nounset # Some useful constants readonly DOCKER_IMAGE_TAG="opensearch-with-ranking-plugin" -readonly OPENSEARCH_VERSION="2.4.0" +readonly OPENSEARCH_VERSION="2.7.0" # # Set default values for OpenSearch (+Dashboards) image tags and plugin URL. @@ -20,14 +20,14 @@ if [ -z "${OPENSEARCH_DASHBOARDS_IMAGE_TAG:-}" ]; then OPENSEARCH_DASHBOARDS_IMAGE_TAG="opensearchproject/opensearch-dashboards:${OPENSEARCH_VERSION}" fi if [ -z "${SEARCH_PROCESSOR_PLUGIN_URL:-}" ]; then - SEARCH_PROCESSOR_PLUGIN_URL="https://github.com/opensearch-project/search-relevance/releases/download/${OPENSEARCH_VERSION}/search-processor.zip" + SEARCH_PROCESSOR_PLUGIN_URL="https://github.com/opensearch-project/search-processor/releases/download/${OPENSEARCH_VERSION}/opensearch-search-processor-${OPENSEARCH_VERSION}.0.zip" fi function print_help() { cat << EOF Usage: $0 [-p ] [-r ] [-e ] [--profile ] [--create-execution-plan] - [--volume-name ] + [--volume-name ] [--admin-password ] -p | --execution-plan-id The ID returned from Kendra Intelligent Ranking service from the call to CreateRescoreExecutionPlan. Required if --create-execution-plan is not set. @@ -50,6 +50,11 @@ Usage: $0 [-p ] [-r ] [-e ] named Docker volume to \$OPENSEARCH_ROOT/data, so index data will persist across executions. If the named volume does not exist, it will be created. + --admin-password For OpenSearch 2.12 and higher, we no longer use a default + password of "admin" for the admin user. Instead, the value + passed to this parameter will be used as the admin password. + For OpenSearch versions prior to 2.12, this argument will be + ignored with a warning. NOTE: If the --profile option is not specified, the script will attempt to read AWS credentials (access/secret key, optional session token) from environment variables, @@ -101,12 +106,18 @@ while [ "$#" -gt 0 ]; do VOLUME_NAME=$1 shift ;; + --admin-password ) + shift + OPENSEARCH_INITIAL_ADMIN_PASSWORD="$1" + shift + ;; esac done FAILED_VALIDATION=0 if [ -z "${EXECUTION_PLAN_ID:-}" ] && [ -z "${CREATE_EXECUTION_PLAN:-}" ]; then - >&2 echo "Missing argument [-p | --execution-plan] or --create-execution-plan" + >&2 echo "Kendra Intelligent Ranking Rescore Execution Plan ID is required. Provide one using the argument [-p | --execution-plan]"\ + " or provision one using --create-execution-plan" FAILED_VALIDATION=1 elif [ -n "${EXECUTION_PLAN_ID:-}" ] && [ -n "${CREATE_EXECUTION_PLAN:-}" ]; then >&2 echo "You cannot specify both [-p | --execution-plan] and --create-execution-plan" @@ -120,6 +131,19 @@ if [ "${FAILED_VALIDATION}" == "1" ]; then exit 1 fi +# Starting in 2.12.0, security demo configuration script requires an initial admin password +OPENSEARCH_REQUIRED_VERSION="2.12.0" +COMPARE_VERSION=`echo $OPENSEARCH_REQUIRED_VERSION $OPENSEARCH_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` +if [ "$COMPARE_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then + if [ -n "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "WARNING: The --admin-password setting has no effect on OpenSearch ${OPENSEARCH_VERSION}. The admin password will be 'admin'." + fi + OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin" +elif [ -z "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "Starting with OpenSearch 2.12, you must specify the admin password with the --admin-password parameter." + exit 1 +fi + # # Determine which credentials and region to use. By the end of this block, all specified # credentials will be loaded into environment variables (or we fail with an explanatory @@ -199,7 +223,8 @@ fi # if [ -z "${KENDRA_RANKING_ENDPOINT:-}" ]; then KENDRA_RANKING_ENDPOINT="https://kendra-ranking.${AWS_REGION}.api.aws" - echo "Missing argument [-e | --kendra-ranking-endpoint] -- using ${KENDRA_RANKING_ENDPOINT}" + echo "Using Kendra Intelligent Ranking endpoint ${KENDRA_RANKING_ENDPOINT}."\ + "To override, use the [-e | --kendra-ranking-endpoint] argument to specify your endpoint URL." fi # @@ -377,6 +402,7 @@ services: - kendra_intelligent_ranking.service.endpoint=${KENDRA_RANKING_ENDPOINT} - kendra_intelligent_ranking.service.region=${AWS_REGION} - kendra_intelligent_ranking.service.execution_plan_id=${EXECUTION_PLAN_ID} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} ulimits: memlock: soft: -1 @@ -444,8 +470,8 @@ cat >README <" https://localhost:9200/ Index some data on OpenSearch by following instructions at https://opensearch.org/docs/latest/opensearch/index-data/ diff --git a/release-notes/search-processor.release-notes-2.4.0.md b/release-notes/search-processor.release-notes-2.4.0.md new file mode 100644 index 0000000..ef37bb5 --- /dev/null +++ b/release-notes/search-processor.release-notes-2.4.0.md @@ -0,0 +1,20 @@ +## Version 2.4.0 Release Notes + +This is the initial release of the search-processor plugin for OpenSearch 2.4.0. +This plugin serves as an extension point to customize search behavior by preprocessing search requests and post-processing search results. +In this first release, the only implementation is a search result post-processor (reranker) that passes results to AWS Kendra Intelligent Ranking. +More implementations and integrations will be added in future releases. Contributions are encouraged! + +### What's Changed + +- Search/relevancy ([#13](https://github.com/opensearch-project/search-processor/pull/13)) +- adding MAINTAINERS.md ([#16](https://github.com/opensearch-project/search-processor/pull/16)) +- Initial code for plugin based on opensearch-plugin-template-java ([#3](https://github.com/opensearch-project/search-processor/pull/3)) +- Merge all changes to 2.x branch before cutting 2.4 branch ([#34](https://github.com/opensearch-project/search-processor/pull/34)) + +### New Contributors +- @YANG-DB made their first contribution in ([#13](https://github.com/opensearch-project/search-processor/pull/13)) +- @kevinawskendra made their first contribution in ([#3](https://github.com/opensearch-project/search-processor/pull/3)) +- @mahitamahesh made their first contribution in ([#23](https://github.com/opensearch-project/search-processor/pull/23)) + +Full Changelog: https://github.com/opensearch-project/search-relevance/commits/2.4.0 \ No newline at end of file diff --git a/release-notes/search-processor.release-notes-2.5.0.md b/release-notes/search-processor.release-notes-2.5.0.md new file mode 100644 index 0000000..11f8b53 --- /dev/null +++ b/release-notes/search-processor.release-notes-2.5.0.md @@ -0,0 +1,36 @@ +Compatible with OpenSearch 2.5.0. + +### Features + +* Add a Docker quickstart script for demo / test ([#25](https://github.com/opensearch-project/search-processor/pull/25)) +* Update log statements in quickstart helper script ([#79](https://github.com/opensearch-project/search-processor/pull/79)) + +### Enhancements + +* Bug fixes and refactoring ([#38](https://github.com/opensearch-project/search-processor/pull/38)) + +### Infrastructure + +* Configure Mend for GitHub.com ([#37](https://github.com/opensearch-project/search-processor/pull/37)) +* Fix codecov after repo renaming ([#41](https://github.com/opensearch-project/search-processor/pull/41)) +* stripped down version of pr stats for only this repository ([#56](https://github.com/opensearch-project/search-processor/pull/56)) +* Fix permissions for PR Stats action ([#67](https://github.com/opensearch-project/search-processor/pull/67)) + +### Documentation + +* Added DEVELOPER_GUIDE.md and html jacoco output ([#35](https://github.com/opensearch-project/search-processor/pull/35)) +* Use short-form MAINTAINERS.md. ([#39](https://github.com/opensearch-project/search-processor/pull/39)) +* Maintainer update and PR stats github action ([#43](https://github.com/opensearch-project/search-processor/pull/43)) +* Updating README.md to talk more about Search Query & Request Transfor… ([#58](https://github.com/opensearch-project/search-processor/pull/58)) +* minor updates to README.md; removed dead link to credits and an unnec… ([#65](https://github.com/opensearch-project/search-processor/pull/65)) +* Updated MAINTAINERS.md format. ([#77](https://github.com/opensearch-project/search-processor/pull/77)) +* adding Mingshi Liu as a maintainer to the repo ([#86](https://github.com/opensearch-project/search-processor/pull/86)) +* Add release-notes folder and 2.4.0 md file ([#89](https://github.com/opensearch-project/search-processor/pull/89)) +* Add release note for 2.5.0 ([#95](https://github.com/opensearch-project/search-processor/pull/95)) + +### Maintenance +* Remove DCO config and update PR template link ([#51](https://github.com/opensearch-project/search-processor/pull/51)) +* Add config rule for auto categorizing labels ([#84](https://github.com/opensearch-project/search-processor/pull/84)) +* Increment from 2.4 to 2.5 and bump jackson from 2.14.0 to 2.14.1 ([#90](https://github.com/opensearch-project/search-processor/pull/90)) + +**Full Changelog**: https://github.com/opensearch-project/search-processor/compare/2.4.0...2.5.0 \ No newline at end of file diff --git a/release-notes/search-processor.release-notes-2.6.0.md b/release-notes/search-processor.release-notes-2.6.0.md new file mode 100644 index 0000000..fa1a22f --- /dev/null +++ b/release-notes/search-processor.release-notes-2.6.0.md @@ -0,0 +1,13 @@ +## Version 2.6.0 Release Notes + +### Enhancements +* Add more test coverage [#100](https://github.com/opensearch-project/search-processor/pull/100) + +### Infrastructure +* Update artifact name [#106](https://github.com/opensearch-project/search-processor/pull/106) + +### Maintenance +* Bumping plugin version to `2.6.0` and `jackson-core` version to `2.14.2` [#110](https://github.com/opensearch-project/search-processor/pull/110) + +### Documentation +* Adding `2.6.0` release notes [#110](https://github.com/opensearch-project/search-processor/pull/110) diff --git a/release-notes/search-processor.release-notes-2.7.0.md b/release-notes/search-processor.release-notes-2.7.0.md new file mode 100644 index 0000000..8ba8b47 --- /dev/null +++ b/release-notes/search-processor.release-notes-2.7.0.md @@ -0,0 +1,11 @@ +## Version 2.7.0 Release Notes + +### Enhancements +* Improve test coverage [#99](https://github.com/opensearch-project/search-processor/pull/99) + +### Infrastructure +* Updating build.gradle to use snapshot by default [#124](https://github.com/opensearch-project/search-processor/pull/124) +* Accommodate changes to XContent classes [#125](https://github.com/opensearch-project/search-processor/pull/125) + +### Documentation +* Prepping for 2.7.0 release [#130](https://github.com/opensearch-project/search-processor/pull/130) diff --git a/release-notes/search-processor.release-notes-2.8.0.md b/release-notes/search-processor.release-notes-2.8.0.md new file mode 100644 index 0000000..287dbce --- /dev/null +++ b/release-notes/search-processor.release-notes-2.8.0.md @@ -0,0 +1,9 @@ + +## Version 2.8.0.0 Release Notes + +Compatible with OpenSearch 2.8.0 + + +### Features +* Add KendraRankingResponseProcessor ([#137](https://github.com/opensearch-project/search-processor/pull/137)) +* Personalized intelligent ranking for open search requests ([#138](https://github.com/opensearch-project/search-processor/pull/138)) \ No newline at end of file diff --git a/release-notes/search-processor.release-notes-2.9.0.md b/release-notes/search-processor.release-notes-2.9.0.md new file mode 100644 index 0000000..8dd944d --- /dev/null +++ b/release-notes/search-processor.release-notes-2.9.0.md @@ -0,0 +1,9 @@ +## Version 2.9.0.0 Release Notes + +Compatible with OpenSearch 2.9.0 + + +### Enhancements +* Support contextual metadata to use when getting personalized reranking ([#144](https://github.com/opensearch-project/search-processor/pull/144)) +* Scoring logic for reranking open search hits based on personalize campaign response ([#147](https://github.com/opensearch-project/search-processor/pull/147)) +* Bring Search Pipeline Update and Add IgnoreFailure and PipelineContext To Each Processors ([#152](https://github.com/opensearch-project/search-processor/pull/152)) diff --git a/settings.gradle b/settings.gradle index f273f7d..96ba866 100644 --- a/settings.gradle +++ b/settings.gradle @@ -7,4 +7,6 @@ * in the user manual at https://docs.gradle.org/6.5.1/userguide/multi_project_builds.html */ -rootProject.name = 'search-relevance' +rootProject.name = 'search-processor' +include 'amazon-kendra-intelligent-ranking' +include 'amazon-personalize-ranking' diff --git a/src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java b/src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java deleted file mode 100644 index 6be5a17..0000000 --- a/src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.search.relevance; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import org.opensearch.action.support.ActionFilter; -import org.opensearch.client.Client; -import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.env.Environment; -import org.opensearch.env.NodeEnvironment; -import org.opensearch.plugins.ActionPlugin; -import org.opensearch.plugins.Plugin; -import org.opensearch.plugins.SearchPlugin; -import org.opensearch.repositories.RepositoriesService; -import org.opensearch.script.ScriptService; -import org.opensearch.search.relevance.actionfilter.SearchActionFilter; -import org.opensearch.search.relevance.transformer.ResultTransformerType; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; -import org.opensearch.search.relevance.client.OpenSearchClient; -import org.opensearch.search.relevance.configuration.SearchConfigurationExtBuilder; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; -import org.opensearch.search.relevance.transformer.ResultTransformer; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; -import org.opensearch.threadpool.ThreadPool; -import org.opensearch.watcher.ResourceWatcherService; - -public class SearchRelevancePlugin extends Plugin implements ActionPlugin, SearchPlugin { - - private OpenSearchClient openSearchClient; - private KendraHttpClient kendraClient; - private KendraIntelligentRanker kendraIntelligentRanker; - - private Map getAllResultTransformers() { - // Initialize and add other transformers here - return Map.of(ResultTransformerType.KENDRA_INTELLIGENT_RANKING, this.kendraIntelligentRanker); - } - - @Override - public List getActionFilters() { - return Arrays.asList(new SearchActionFilter(getAllResultTransformers(), openSearchClient)); - } - - @Override - public List> getSettings() { - // NOTE: cannot use kendraIntelligentRanker.getTransformerSettings because the object is not yet created - List> allTransformerSettings = new ArrayList<>(); - allTransformerSettings.addAll(KendraIntelligentRankerSettings.getAllSettings()); - // Add settings for other transformers here - return allTransformerSettings; - } - - @Override - public Collection createComponents( - Client client, - ClusterService clusterService, - ThreadPool threadPool, - ResourceWatcherService resourceWatcherService, - ScriptService scriptService, - NamedXContentRegistry xContentRegistry, - Environment environment, - NodeEnvironment nodeEnvironment, - NamedWriteableRegistry namedWriteableRegistry, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier - ) { - this.openSearchClient = new OpenSearchClient(client); - this.kendraClient = new KendraHttpClient(KendraClientSettings.getClientSettings(environment.settings())); - this.kendraIntelligentRanker = new KendraIntelligentRanker(this.kendraClient); - - return Arrays.asList( - this.openSearchClient, - this.kendraClient, - this.kendraIntelligentRanker - ); - } - - @Override - public List> getSearchExts() { - return Collections.singletonList( - new SearchExtSpec<>(SearchConfigurationExtBuilder.NAME, SearchConfigurationExtBuilder::new, SearchConfigurationExtBuilder::parse)); - } - -} diff --git a/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java b/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java deleted file mode 100644 index 328ab39..0000000 --- a/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.search.relevance.configuration; - -import static org.opensearch.search.relevance.configuration.Constants.RESULT_TRANSFORMER_SETTING_PREFIX; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.common.settings.Settings; -import org.opensearch.search.SearchExtBuilder; -import org.opensearch.search.relevance.transformer.ResultTransformerType; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfiguration; - -public class ConfigurationUtils { - - /** - * Get result transformer configurations from Search Request - * @param settings all index settings configured for this plugin - * @return ordered and validated list of result transformers, empty list if not specified - */ - public static List getResultTransformersFromIndexConfiguration( - Settings settings) { - List indexLevelConfigs = new ArrayList<>(); - - if (settings != null) { - if (settings.getGroups(RESULT_TRANSFORMER_SETTING_PREFIX) != null) { - for (Map.Entry resultTransformer : settings.getGroups(RESULT_TRANSFORMER_SETTING_PREFIX).entrySet()) { - ResultTransformerType resultTransformerType = ResultTransformerType.fromString(resultTransformer.getKey()); - if (ResultTransformerType.KENDRA_INTELLIGENT_RANKING.equals(resultTransformerType)) { - indexLevelConfigs.add(new KendraIntelligentRankingConfiguration(resultTransformer.getValue())); - } - } - } - } - - return reorderAndValidateConfigs(indexLevelConfigs); - } - - /** - * Get result transformer configurations from Search Request - * @param searchRequest input request - * @return ordered and validated list of result transformers, empty list if not specified - */ - public static List getResultTransformersFromRequestConfiguration( - final SearchRequest searchRequest) { - - // Fetch result transformers specified in request - SearchConfigurationExtBuilder requestLevelSearchConfiguration = null; - if (searchRequest.source() != null && searchRequest.source().ext() != null && !searchRequest.source().ext().isEmpty()) { - // Filter ext builders by name - List extBuilders = searchRequest.source().ext().stream() - .filter(searchExtBuilder -> SearchConfigurationExtBuilder.NAME.equals(searchExtBuilder.getWriteableName())) - .collect(Collectors.toList()); - if (!extBuilders.isEmpty()) { - requestLevelSearchConfiguration = (SearchConfigurationExtBuilder) extBuilders.get(0); - } - } - - List requestLevelConfigs = new ArrayList<>(); - if (requestLevelSearchConfiguration != null) { - requestLevelConfigs = reorderAndValidateConfigs(requestLevelSearchConfiguration.getResultTransformers()); - } - return requestLevelConfigs; - } - - /** - * Sort configurations in ascending order of invocation, and validate - * @param configs list of result transformer configurations - * @return ordered and validated list of result transformers - */ - public static List reorderAndValidateConfigs( - final List configs) throws IllegalArgumentException { - - // Sort - configs.sort(Comparator.comparingInt(ResultTransformerConfiguration::getOrder)); - - for (int i = 0; i < configs.size(); ++i) { - if (configs.get(i).getOrder() != (i + 1)) { - throw new IllegalArgumentException("Expected order [" + (i + 1) + "] for transformer [" + - configs.get(i).getType() + "], but found [" + configs.get(i).getOrder() + "]"); - } - } - - return configs; - } -} diff --git a/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java b/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java deleted file mode 100644 index 170e934..0000000 --- a/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.search.relevance.configuration; - -import static org.opensearch.search.relevance.configuration.Constants.SEARCH_CONFIGURATION; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.common.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; -import org.opensearch.search.SearchExtBuilder; -import org.opensearch.search.relevance.transformer.ResultTransformerType; -import org.opensearch.search.relevance.transformer.TransformerType; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfiguration; - -public class SearchConfigurationExtBuilder extends SearchExtBuilder { - public static final String NAME = SEARCH_CONFIGURATION; - - private static final ParseField RESULT_TRANSFORMER = new ParseField(TransformerType.RESULT_TRANSFORMER.toString()); - private static final ParseField KENDRA_INTELLIGENT_RANKING = new ParseField(ResultTransformerType.KENDRA_INTELLIGENT_RANKING.toString()); - - private List resultTransformerConfigurations = new ArrayList<>(); - - public SearchConfigurationExtBuilder() {} - - public SearchConfigurationExtBuilder(StreamInput input) throws IOException { - ResultTransformerConfiguration cfg1 = input.readOptionalWriteable(KendraIntelligentRankingConfiguration::new); - resultTransformerConfigurations.add(cfg1); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - for (ResultTransformerConfiguration config : resultTransformerConfigurations) { - out.writeOptionalWriteable(config); - } - } - - @Override - public String getWriteableName() { - return NAME; - } - - public static SearchConfigurationExtBuilder parse(XContentParser parser) throws IOException { - SearchConfigurationExtBuilder extBuilder = new SearchConfigurationExtBuilder(); - XContentParser.Token token = parser.currentToken(); - String currentFieldName = null; - if (token != XContentParser.Token.START_OBJECT && (token = parser.nextToken()) != XContentParser.Token.START_OBJECT) { - throw new ParsingException( - parser.getTokenLocation(), - "Expected [" + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", - parser.getTokenLocation() - ); - } - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if (RESULT_TRANSFORMER.match(currentFieldName, parser.getDeprecationHandler())) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } - if (KENDRA_INTELLIGENT_RANKING.match(currentFieldName, - parser.getDeprecationHandler())) { - extBuilder.addResultTransformer( - KendraIntelligentRankingConfiguration.parse(parser, null)); - } else { - throw new IllegalArgumentException( - "Unrecognized Result Transformer type [" + currentFieldName + "]"); - } - } - } else { - throw new IllegalArgumentException("Unrecognized Transformer type [" + currentFieldName + "]"); - } - } else { - throw new ParsingException( - parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + currentFieldName + "].", - parser.getTokenLocation() - ); - } - } - return extBuilder; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); - for (ResultTransformerConfiguration config : resultTransformerConfigurations) { - builder.field(config.getType().toString(), config); - } - return builder.endObject(); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (!(obj instanceof SearchConfigurationExtBuilder)) { - return false; - } - SearchConfigurationExtBuilder o = (SearchConfigurationExtBuilder) obj; - return (this.resultTransformerConfigurations.size() == o.resultTransformerConfigurations.size() && - this.resultTransformerConfigurations.containsAll(o.resultTransformerConfigurations) && - o.resultTransformerConfigurations.containsAll(this.resultTransformerConfigurations)) ; - } - - @Override - public int hashCode() { - return Objects.hash(this.getClass(), this.resultTransformerConfigurations); - } - - public SearchConfigurationExtBuilder setResultTransformers(final List resultTransformerConfigurations) { - this.resultTransformerConfigurations = resultTransformerConfigurations; - return this; - } - - public List getResultTransformers() { - return this.resultTransformerConfigurations; - } - - public void addResultTransformer(final ResultTransformerConfiguration resultTransformerConfiguration) { - this.resultTransformerConfigurations.add(resultTransformerConfiguration); - } -} diff --git a/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformerType.java b/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformerType.java deleted file mode 100644 index 81880fc..0000000 --- a/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformerType.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.search.relevance.transformer; - -public enum ResultTransformerType { - KENDRA_INTELLIGENT_RANKING("kendra_intelligent_ranking"); - - private final String type; - - ResultTransformerType(String type) { - this.type = type; - } - - @Override - public String toString() { - return type; - } - - public static ResultTransformerType fromString(String type) { - for (ResultTransformerType resultTransformerType : values()) { - if (resultTransformerType.type.equalsIgnoreCase(type)) { - return resultTransformerType; - } - } - throw new IllegalArgumentException("Unrecognized Result Transformer type [" + type + "]"); - } -} diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java b/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java deleted file mode 100644 index 3093cc5..0000000 --- a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ -package org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration; - -import org.junit.Test; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; - -public class KendraIntelligentRankingConfigurationTests extends OpenSearchTestCase { - public void testParseWithNullParserAndContext() { - try { - KendraIntelligentRankingConfiguration.parse(null, null); - fail(); - } catch (NullPointerException | IOException e) { - } - } -} \ No newline at end of file diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml deleted file mode 100644 index 7779650..0000000 --- a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ /dev/null @@ -1,8 +0,0 @@ -"Test that the plugin is loaded in OpenSearch": - - do: - cat.plugins: - local: true - h: component - - - match: - $body: /^search-processor\n$/