diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 61bfc57..6a31d4d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,7 +15,7 @@ jobs: Build-search-request-processor: strategy: matrix: - java: [11, 17] + java: [21] os: [ubuntu-latest, macos-latest, windows-latest] name: Build and Test Search Request Processor Plugin @@ -23,10 +23,10 @@ jobs: steps: - name: Checkout Search Request Processor - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} @@ -42,7 +42,7 @@ jobs: - name: Upload Coverage Report if: ${{matrix.os}} == 'ubuntu' - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@29386c70ef20e286228c72b668a06fd0e8399192 # v1 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml index 15b9a55..846441e 100644 --- a/.github/workflows/add-untriaged.yml +++ b/.github/workflows/add-untriaged.yml @@ -4,11 +4,14 @@ on: issues: types: [opened, reopened, transferred] +permissions: + issues: write + jobs: apply-label: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 with: script: | github.rest.issues.addLabels({ diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index e47d8d8..2609a3c 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,14 +15,14 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Backport - uses: VachaShah/backport@v1.1.4 + uses: VachaShah/backport@28c49d91ceec57d7c9f625f1031c1a4d637251f5 # v1.1.4 with: github_token: ${{ steps.github_app_token.outputs.token }} branch_name: backport/backport-${{ github.event.number }} diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 3a7e88c..8644c7c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -13,7 +13,7 @@ jobs: Restart-Upgrade-BWCTests-k-NN: strategy: matrix: - java: [ 11, 17 ] + java: [ 21 ] bwc_version : [ "2.7.0" ] opensearch_version : [ "3.0.0-SNAPSHOT" ] @@ -24,10 +24,10 @@ jobs: steps: - name: Checkout SRP - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} @@ -41,8 +41,8 @@ jobs: Rolling-Upgrade-BWCTests-SRP: strategy: matrix: - java: [ 11, 17 ] - bwc_version: [ "2.7.0" ] + java: [ 21 ] + bwc_version: [ "2.11.1" ] opensearch_version: [ "3.0.0-SNAPSHOT" ] name: SRP Rolling-Upgrade BWC Tests @@ -52,10 +52,10 @@ jobs: steps: - name: Checkout SRP - uses: actions/checkout@v1 + uses: actions/checkout@50fbc622fc4ef5163becd7fab6573eac35f8462e # v1 - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@b6e674f4b717d7b0ae3baee0fbe79f498905dfde # v1 with: java-version: ${{ matrix.java }} diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml index cb1eb40..3938005 100644 --- a/.github/workflows/create-documentation-issue.yml +++ b/.github/workflows/create-documentation-issue.yml @@ -14,14 +14,14 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: Edit the issue template run: | @@ -29,7 +29,7 @@ jobs: - name: Create Issue From File id: create-issue - uses: peter-evans/create-issue-from-file@v4 + uses: peter-evans/create-issue-from-file@433e51abf769039ee20ba1293a088ca19d573b7f # v4 with: title: Add documentation related to new feature content-filepath: ./.github/ISSUE_TEMPLATE/documentation-issue.md diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index a97f9cd..156d749 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -10,6 +10,6 @@ jobs: if: startsWith(github.event.pull_request.head.ref,'backport-') steps: - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main + uses: SvanBoxel/delete-merged-branch@2b5b058e3db41a3328fd9a6a58fd4c2545a14353 # main env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml index bdafb18..7d13551 100644 --- a/.github/workflows/dependabot_pr.yml +++ b/.github/workflows/dependabot_pr.yml @@ -11,14 +11,14 @@ jobs: steps: - name: GitHub App token id: github_app_token - uses: tibdex/github-app-token@v1.5.0 + uses: tibdex/github-app-token@1901dc7d52169e70c27a8da37aef0d423e2867a2 # v1.5.0 with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} installation_id: 22958780 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 with: token: ${{ steps.github_app_token.outputs.token }} @@ -27,7 +27,7 @@ jobs: ./gradlew updateSHAs - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4.7.2 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.7.2 with: commit_message: Updating SHAs branch: ${{ github.head_ref }} @@ -36,7 +36,7 @@ jobs: commit_options: '--signoff' - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4.7.2 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4.7.2 with: commit_message: Spotless formatting branch: ${{ github.head_ref }} @@ -45,12 +45,12 @@ jobs: commit_options: '--signoff' - name: Update the changelog - uses: dangoslen/dependabot-changelog-helper@v1 + uses: dangoslen/dependabot-changelog-helper@780f7c82213ff956b1bd8cb484ba67d1fbe8b4ba # v1 with: version: 'Unreleased' - name: Commit the changes - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@3ea6ae190baf489ba007f7c92608f33ce20ef04a # v4 with: commit_message: "Update changelog" branch: ${{ github.head_ref }} diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 6b3d89c..48e2bf9 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update draft release notes - uses: release-drafter/release-drafter@v5 + uses: release-drafter/release-drafter@09c613e259eb8d4e7c81c2cb00618eb5fc4575a7 # v5 with: config-name: draft-release-notes-config.yml name: Version (set here) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 3d0b81a..88f0d32 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: lychee Link Checker id: lychee - uses: lycheeverse/lychee-action@master + uses: lycheeverse/lychee-action@6da1d14f3a43098a294b7696d93d938aa8d20fc0 # master with: args: --accept=200,403,429 **/*.html **/*.md **/*.txt **/*.json env: diff --git a/.github/workflows/pr_stats.yml b/.github/workflows/pr_stats.yml index 96c971b..5cb0564 100644 --- a/.github/workflows/pr_stats.yml +++ b/.github/workflows/pr_stats.yml @@ -12,4 +12,4 @@ jobs: pull-requests: write steps: - name: Run pull request stats - uses: flowwer-dev/pull-request-stats@master + uses: flowwer-dev/pull-request-stats@0dde6edf8b7db75684533021212c6e85fb987dbb # master diff --git a/README.md b/README.md index 5842249..06b5ecc 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![codecov](https://codecov.io/gh/opensearch-project/search-processor/branch/main/graph/badge.svg?token=PYQO2GW39S)](https://codecov.io/gh/opensearch-project/search-processor) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) -# Search Query & Request Transformers +# Search Rerankers: AWS Kendra & AWS Personalize - [Welcome!](#welcome) - [Project Resources](#project-resources) - [Code of Conduct](#code-of-conduct) @@ -10,21 +10,21 @@ - [Copyright](#copyright) ## Welcome! -This repository is the home of an evolving project that aims to create a pipeline of transformers to preprocess queries before search and post-process results after search. The first component here is a plugin to re-rank search results before returning them to the client inline. In the coming year, we will add hooks to configure other re-rankers and allow users to add their own components to the pipeline. Logging will also be a critical part of the pipeline in two ways: -1. Logging information about the search experience (e.g. query, search results returned from the index, search results returned to the OpenSearch client) -1. Logging debug information about the transformers +This repository hosts the code for two self-install re-rankers that integrate into [Search Pipelines](https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/). User documentation for the Personalize Reranker is [here](https://opensearch.org/docs/latest/search-plugins/search-pipelines/personalize-search-ranking/). For Kendra, it is [here](https://opensearch.org/docs/latest/search-plugins/search-relevance/index/#reranking-results-with-kendra-intelligent-ranking-for-opensearch). + +# Search Processors: Where Do They Go? +The current guideline for developing processors is that if you are developing a processor that would introduce new dependencies in [OpenSearch Core](https://github.com/opensearch-project/OpenSearch) (e.g. new libraries, makes a network connection outside of OpenSearch), it should be in a separate repository. Please consider creating it in a standalone repository since each processor should be thought of like a \*NIX command with input and output connected by pipes (i.e. a Search Pipeline). Each processor should do one thing and do it well. Otherwise, it could go into the OpenSearch repository under [org.opensearch.search.pipeline.common](https://github.com/opensearch-project/OpenSearch/tree/a08d588691c3b232e65d73b0a0c2fc5c72c870cf/modules/search-pipeline-common). If you have doubts, just create an issue in OpenSearch Core and, if you have one, a new PR. Maintainers will help guide you. -We will be publishing an RFC soon to give more detail and have a deeper conversation, but for now take a look at the code, open issues, comment, etc. # History This repository has also been used for discussion and ideas around search relevance. These discussions still exist here, however due to the relatively new standard of having one repo per plugin in OpenSearch and our implementations beginning to make it into the OpenSearch build, we have two repositories now. This repository will develop into a plugin that will allow OpenSearch users to rewrite search queries, rerank results, and log data about those actions. The other repository, [dashboards-search-relevance](https://www.github.com/opensearch-projects/dashboards-search-relevance), is where we will build front-end tooling to help relevance engineers and business users tune results. - ## Project Resources * [OpenSearch Project Website](https://opensearch.org/) * [Downloads](https://opensearch.org/downloads.html) * [Project Principles](https://opensearch.org/#principles) +* [Search Pipelines](https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/) * [Contributing to OpenSearch Search Request Processor](CONTRIBUTING.md) * [Search Relevance](RELEVANCE.md) * [Maintainer Responsibilities](MAINTAINERS.md) diff --git a/amazon-kendra-intelligent-ranking/build.gradle b/amazon-kendra-intelligent-ranking/build.gradle new file mode 100644 index 0000000..60821f6 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/build.gradle @@ -0,0 +1,118 @@ +import org.opensearch.gradle.test.RestIntegTestTask + +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'jacoco' + +group = 'org.opensearch' + +def pluginName = 'amazon-kendra-intelligent-ranking' +def pluginDescription = 'Rerank search results using Amazon Kendra Intelligent Ranking' +def projectPath = 'org.opensearch' +def pathToPlugin = 'search.relevance' +def pluginClassName = 'AmazonKendraIntelligentRankingPlugin' + +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}.0" + version "${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') +} + +java { + targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 +} + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + +buildscript { + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + implementation 'com.ibm.icu:icu4j:57.2' + implementation 'org.apache.httpcomponents:httpclient:4.5.14' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2' + implementation 'commons-logging:commons-logging:1.2' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' +} + + +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' + } +} + + +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +testClusters.integTest { + testDistribution = "ARCHIVE" + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) +} + +run { + useCluster testClusters.integTest +} + +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false diff --git a/src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java similarity index 83% rename from src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java index 1f14ca1..ef9df5d 100644 --- a/src/main/java/org/opensearch/search/relevance/SearchRelevancePlugin.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPlugin.java @@ -11,8 +11,8 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; @@ -27,17 +27,14 @@ import org.opensearch.search.relevance.actionfilter.SearchActionFilter; import org.opensearch.search.relevance.client.OpenSearchClient; import org.opensearch.search.relevance.configuration.ResultTransformerConfigurationFactory; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; import org.opensearch.search.relevance.configuration.SearchConfigurationExtBuilder; -import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; import org.opensearch.search.relevance.transformer.ResultTransformer; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.KendraIntelligentRanker; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraClientSettings; +import org.opensearch.search.relevance.transformer.kendraintelligentranking.client.KendraHttpClient; import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankerSettings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.configuration.KendraIntelligentRankingConfigurationFactory; import org.opensearch.search.relevance.transformer.kendraintelligentranking.pipeline.KendraRankingResponseProcessor; -import org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor; -import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; -import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParametersExtBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; @@ -49,7 +46,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -public class SearchRelevancePlugin extends Plugin implements ActionPlugin, SearchPlugin, SearchPipelinePlugin { +public class AmazonKendraIntelligentRankingPlugin extends Plugin implements ActionPlugin, SearchPlugin, SearchPipelinePlugin { private OpenSearchClient openSearchClient; private KendraHttpClient kendraClient; @@ -112,15 +109,11 @@ public List> getSearchExts() { .collect(Collectors.toMap(ResultTransformerConfigurationFactory::getName, i -> i)); return List.of(new SearchExtSpec<>(SearchConfigurationExtBuilder.NAME, input -> new SearchConfigurationExtBuilder(input, resultTransformerMap), - parser -> SearchConfigurationExtBuilder.parse(parser, resultTransformerMap)), - new SearchExtSpec<>(PersonalizeRequestParametersExtBuilder.NAME, - input -> new PersonalizeRequestParametersExtBuilder(input), - parser -> PersonalizeRequestParametersExtBuilder.parse(parser))); + parser -> SearchConfigurationExtBuilder.parse(parser, resultTransformerMap))); } @Override public Map> getResponseProcessors(Parameters parameters) { - return Map.of(PersonalizeRankingResponseProcessor.TYPE, new PersonalizeRankingResponseProcessor.Factory(PersonalizeClientSettings.getClientSettings(parameters.env.settings())), - KendraRankingResponseProcessor.TYPE, new KendraRankingResponseProcessor.Factory(this.kendraClientSettings)); + return Map.of(KendraRankingResponseProcessor.TYPE, new KendraRankingResponseProcessor.Factory(this.kendraClientSettings)); } } \ No newline at end of file diff --git a/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java similarity index 99% rename from src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java index 1b2420f..229172c 100644 --- a/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/actionfilter/SearchActionFilter.java @@ -10,9 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; -import org.opensearch.action.ActionListener; import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionResponse; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -20,6 +18,8 @@ import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java similarity index 96% rename from src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java index 438685d..42baf89 100644 --- a/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/client/OpenSearchClient.java @@ -7,12 +7,12 @@ */ package org.opensearch.search.relevance.client; -import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; +import org.opensearch.core.action.ActionListener; public class OpenSearchClient { private final Client client; diff --git a/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java similarity index 99% rename from src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java index 3f22951..44081d6 100644 --- a/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ConfigurationUtils.java @@ -7,7 +7,6 @@ */ package org.opensearch.search.relevance.configuration; -import org.opensearch.action.ActionListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.common.settings.Settings; import org.opensearch.search.SearchExtBuilder; diff --git a/src/main/java/org/opensearch/search/relevance/configuration/Constants.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/Constants.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/Constants.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/Constants.java diff --git a/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfiguration.java diff --git a/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/ResultTransformerConfigurationFactory.java diff --git a/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilder.java diff --git a/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/configuration/TransformerConfiguration.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/ResultTransformer.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/TransformerType.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRanker.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettings.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java similarity index 96% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java index 64c2a82..11b44dd 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java +++ b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClient.java @@ -30,13 +30,12 @@ import java.io.ByteArrayInputStream; import java.io.Closeable; -import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; -import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.Strings; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreRequest; import org.opensearch.search.relevance.transformer.kendraintelligentranking.model.dto.RescoreResult; @@ -56,6 +55,7 @@ public class KendraHttpClient implements Closeable { private final ObjectMapper objectMapper = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + @SuppressWarnings("removal") public KendraHttpClient(KendraClientSettings clientSettings) { serviceEndpoint = clientSettings.getServiceEndpoint(); executionPlanId = clientSettings.getExecutionPlanId(); @@ -103,6 +103,7 @@ public KendraHttpClient(KendraClientSettings clientSettings) { } } + @SuppressWarnings({ "deprecation", "removal" }) public RescoreResult rescore(RescoreRequest rescoreRequest) { return AccessController.doPrivileged((PrivilegedAction) () -> { try { @@ -132,11 +133,11 @@ public URI buildRescoreURI() { } public boolean isValid() { - return StringUtils.isNotEmpty(serviceEndpoint) && StringUtils.isNotEmpty(executionPlanId); + return !Strings.isNullOrEmpty(serviceEndpoint) && !Strings.isNullOrEmpty(executionPlanId); } @Override - public void close() throws IOException { + public void close() { if (amazonHttpClient != null) { amazonHttpClient.shutdown(); } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandler.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleResponseHandler.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/Constants.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankerSettings.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfiguration.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationFactory.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingException.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/PassageScore.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/Document.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreRequest.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreRequest.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreRequest.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreRequest.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResult.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResult.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResult.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResult.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResultItem.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResultItem.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResultItem.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/RescoreResultItem.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessor.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessor.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessor.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessor.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25Scorer.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGenerator.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParser.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitter.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java b/amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java rename to amazon-kendra-intelligent-ranking/src/main/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizer.java diff --git a/src/main/plugin-metadata/plugin-security.policy b/amazon-kendra-intelligent-ranking/src/main/plugin-metadata/plugin-security.policy similarity index 100% rename from src/main/plugin-metadata/plugin-security.policy rename to amazon-kendra-intelligent-ranking/src/main/plugin-metadata/plugin-security.policy diff --git a/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java new file mode 100644 index 0000000..cc13e70 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingPluginIT.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import java.io.IOException; + +public class AmazonKendraIntelligentRankingPluginIT extends OpenSearchRestTestCase { + + public void testPluginInstalled() throws IOException, ParseException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins")); + String body = EntityUtils.toString(response.getEntity()); + + logger.info("response body: {}", body); + assertNotNull(body); + assertTrue(body.contains("amazon-kendra-intelligent-ranking")); + } +} diff --git a/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/SearchRelevanceTests.java diff --git a/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java similarity index 99% rename from src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java index ac764d4..496adbf 100644 --- a/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java +++ b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/actionfilter/SearchActionFilterTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.search.TotalHits; import org.mockito.Mockito; -import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.settings.get.GetSettingsAction; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -24,6 +23,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.client.Client; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.document.DocumentField; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/configuration/SearchConfigurationExtBuilderTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/KendraIntelligentRankerTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraClientSettingsTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraHttpClientTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/KendraIntelligentClientTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/client/SimpleAwsErrorHandlerTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/configuration/KendraIntelligentRankingConfigurationTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/KendraIntelligentRankingExceptionTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/model/dto/DocumentTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/pipeline/KendraRankingResponseProcessorTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/BM25ScorerTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/PassageGeneratorTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/QueryParserTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/SentenceSplitterTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java b/amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java rename to amazon-kendra-intelligent-ranking/src/test/java/org/opensearch/search/relevance/transformer/kendraintelligentranking/preprocess/TextTokenizerTests.java diff --git a/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java b/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java new file mode 100644 index 0000000..ae37662 --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonKendraIntelligentRankingClientYamlTestSuiteIT.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.opensearch.test.rest.yaml.ClientYamlTestCandidate; +import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; + + +public class AmazonKendraIntelligentRankingClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { + + public AmazonKendraIntelligentRankingClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return OpenSearchClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml new file mode 100644 index 0000000..af5e5fe --- /dev/null +++ b/amazon-kendra-intelligent-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -0,0 +1,17 @@ +"Test that the plugin is loaded in OpenSearch": + - do: + cat.plugins: + local: true + h: component + + - match: + $body: /^opensearch-amazon-kendra-intelligent-ranking-\d+.\d+.\d+.\d+\n$/ + + - do: + indices.create: + index: test + + - do: + search: + index: test + body: { } diff --git a/amazon-personalize-ranking/build.gradle b/amazon-personalize-ranking/build.gradle new file mode 100644 index 0000000..527d985 --- /dev/null +++ b/amazon-personalize-ranking/build.gradle @@ -0,0 +1,129 @@ +import org.opensearch.gradle.test.RestIntegTestTask + +apply plugin: 'java' +apply plugin: 'idea' +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'jacoco' + +group = 'org.opensearch' + +def pluginName = 'amazon-personalize-ranking' +def pluginDescription = 'Rerank search results using Amazon Personalize' +def projectPath = 'org.opensearch' +def pathToPlugin = 'search.relevance' +def pluginClassName = 'AmazonPersonalizeRankingPlugin' + + +opensearchplugin { + name "opensearch-${pluginName}-${plugin_version}.0" + version "${plugin_version}" + description pluginDescription + classname "${projectPath}.${pathToPlugin}.${pluginClassName}" + licenseFile rootProject.file('LICENSE') + noticeFile rootProject.file('NOTICE') +} + +java { + targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 +} + +// This requires an additional Jar not published as part of build-tools +loggerUsageCheck.enabled = false + +// No need to validate pom, as we do not upload to maven/sonatype +validateNebulaPom.enabled = false + +buildscript { + repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + } + + dependencies { + classpath "org.opensearch.gradle:build-tools:${opensearch_version}" + } +} + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + implementation 'org.apache.httpcomponents:httpclient:4.5.14' + implementation 'org.apache.httpcomponents:httpcore:4.4.16' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.18.2' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2' + implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' + implementation 'com.amazonaws:aws-java-sdk-personalizeruntime:1.12.300' + implementation 'commons-logging:commons-logging:1.2' +} + + +allprojects { + plugins.withId('jacoco') { + jacoco.toolVersion = '0.8.9' + } +} + + +test { + include '**/*Tests.class' + finalizedBy jacocoTestReport +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +integTest { + // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' + } +} + +testClusters.integTest { + testDistribution = "ARCHIVE" + + // This installs our plugin into the testClusters + plugin(project.tasks.bundlePlugin.archiveFile) +} + +run { + useCluster testClusters.integTest +} + +sourceSets { + main { + resources { + srcDirs = ["config"] + includes = ["**/*.yml"] + } + } +} + + +jacocoTestReport { + dependsOn test + reports { + xml.required = true + html.required = true + } +} + +// TODO: Enable these checks +dependencyLicenses.enabled = false +thirdPartyAudit.enabled = false +loggerUsageCheck.enabled = false diff --git a/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java new file mode 100644 index 0000000..343fae5 --- /dev/null +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPlugin.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.search.relevance; + +import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Setting; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchPipelinePlugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.script.ScriptService; +import org.opensearch.search.pipeline.Processor; +import org.opensearch.search.pipeline.SearchResponseProcessor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParametersExtBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class AmazonPersonalizeRankingPlugin extends Plugin implements SearchPlugin, SearchPipelinePlugin { + + private PersonalizeClientSettings personalizeClientSettings; + + @Override + public List> getSettings() { + // Add settings for other transformers here + return new ArrayList<>(PersonalizeClientSettings.getAllSettings()); + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + this.personalizeClientSettings = PersonalizeClientSettings.getClientSettings(environment.settings()); + + return Collections.emptyList(); + } + + @Override + public List> getSearchExts() { + return List.of( + new SearchPlugin.SearchExtSpec<>(PersonalizeRequestParametersExtBuilder.NAME, + PersonalizeRequestParametersExtBuilder::new, + PersonalizeRequestParametersExtBuilder::parse)); + } + + @Override + public Map> getResponseProcessors(Parameters parameters) { + return Map.of(PersonalizeRankingResponseProcessor.TYPE, new PersonalizeRankingResponseProcessor.Factory(this.personalizeClientSettings)); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java similarity index 89% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java index 3bbc9d6..670915c 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessor.java @@ -154,7 +154,7 @@ public Factory(PersonalizeClientSettings settings) { } @Override - public PersonalizeRankingResponseProcessor create(Map> processorFactories, String tag, String description, boolean ignoreFailure, Map config, PipelineContext pipelineContext) throws Exception { + public PersonalizeRankingResponseProcessor create(Map> processorFactories, String tag, String description, boolean ignoreFailure, Map config, PipelineContext pipelineContext) { String personalizeCampaign = ConfigurationUtils.readStringProperty(TYPE, tag, config, CAMPAIGN_ARN_CONFIG_NAME); String iamRoleArn = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, IAM_ROLE_ARN_CONFIG_NAME); String recipe = ConfigurationUtils.readStringProperty(TYPE, tag, config, RECIPE_CONFIG_NAME); @@ -165,9 +165,25 @@ public PersonalizeRankingResponseProcessor create(Map) () -> new ClientConfiguration() + .withUserAgentPrefix(USER_AGENT_PREFIX)); personalizeRuntime = AccessController.doPrivileged( (PrivilegedAction) () -> AmazonPersonalizeRuntimeClientBuilder.standard() .withCredentials(credentialsProvider) diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java similarity index 93% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java index 9ecf606..1a019b5 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettings.java @@ -18,6 +18,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsException; +import java.util.Arrays; +import java.util.Collection; + /** * Container for personalize client settings such as AWS credentials */ @@ -46,6 +49,14 @@ protected PersonalizeClientSettings(AWSCredentials credentials) { this.credentials = credentials; } + public static Collection> getAllSettings() { + return Arrays.asList( + ACCESS_KEY_SETTING, + SECRET_KEY_SETTING, + SESSION_TOKEN_SETTING + ); + } + public AWSCredentials getCredentials() { return credentials; } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactory.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java similarity index 82% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java index eefea08..40105f9 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/Constants.java @@ -13,4 +13,5 @@ */ public class Constants { public static final String AMAZON_PERSONALIZED_RANKING_RECIPE_NAME = "aws-personalized-ranking"; + public static final String AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME = "aws-personalized-ranking-v2"; } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfiguration.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtil.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameters.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilder.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java similarity index 100% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRanker.java diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java similarity index 83% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java index 1e30ec9..ead8dd8 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/PersonalizedRankerFactory.java @@ -14,6 +14,7 @@ import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl.AmazonPersonalizedRankerImpl; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; /** * Factory for creating Personalize ranker instance based on Personalize ranker configuration @@ -29,7 +30,9 @@ public class PersonalizedRankerFactory { */ public PersonalizedRanker getPersonalizedRanker(PersonalizeIntelligentRankerConfiguration config, PersonalizeClient client){ PersonalizedRanker ranker = null; - if (config.getRecipe().equals(AMAZON_PERSONALIZED_RANKING_RECIPE_NAME)) { + String recipeInConfig = config.getRecipe(); + if (recipeInConfig.equals(AMAZON_PERSONALIZED_RANKING_RECIPE_NAME) + || recipeInConfig.equals(AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME)) { ranker = new AmazonPersonalizedRankerImpl(config, client); } else { logger.error("Personalize recipe provided in configuration is not supported for re ranking search results"); diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java similarity index 94% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java index cb8cac1..142dceb 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/reranker/impl/AmazonPersonalizedRankerImpl.java @@ -7,6 +7,7 @@ */ package org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.impl; +import com.amazonaws.AmazonServiceException; import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingRequest; import com.amazonaws.services.personalizeruntime.model.GetPersonalizedRankingResult; import com.amazonaws.services.personalizeruntime.model.PredictedItem; @@ -20,6 +21,7 @@ import org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.PersonalizeIntelligentRankerConfiguration; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.reranker.PersonalizedRanker; +import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.ValidationUtil; import java.util.ArrayList; import java.util.Arrays; @@ -36,6 +38,7 @@ public class AmazonPersonalizedRankerImpl implements PersonalizedRanker { private static final Logger logger = LogManager.getLogger(AmazonPersonalizedRankerImpl.class); private final PersonalizeIntelligentRankerConfiguration rankerConfig; private final PersonalizeClient personalizeClient; + public AmazonPersonalizedRankerImpl(PersonalizeIntelligentRankerConfiguration config, PersonalizeClient client) { this.rankerConfig = config; @@ -96,7 +99,15 @@ public SearchHits rerank(SearchHits hits, PersonalizeRequestParameters requestPa SearchHits personalizedHits = combineScores(hits, result); return personalizedHits; - } catch (Exception ex) { + } catch (AmazonServiceException e) { + logger.error("Exception while calling personalize campaign: {}", e.getMessage()); + int statusCode = e.getStatusCode(); + if (ValidationUtil.is4xxError(statusCode)) { + throw new IllegalArgumentException(e); + } + throw e; + } + catch (Exception ex) { logger.error("Failed to re rank with Personalize.", ex); throw ex; } diff --git a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java similarity index 86% rename from src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java rename to amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java index c318d46..c6c92f7 100644 --- a/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java +++ b/amazon-personalize-ranking/src/main/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtil.java @@ -15,9 +15,13 @@ import java.util.HashSet; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; public class ValidationUtil { - private static Set SUPPORTED_PERSONALIZE_RECIPES = new HashSet<>(Arrays.asList(AMAZON_PERSONALIZED_RANKING_RECIPE_NAME)); + private static Set SUPPORTED_PERSONALIZE_RECIPES = new HashSet<>(Arrays.asList( + AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, + AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME + )); /** * Validate Personalize configuration for calling Personalize service. @@ -57,4 +61,11 @@ private static boolean isValidCampaignOrRoleArn(String arn, String expectedServi return false; } } + + public static boolean is4xxError(int statusCode){ + if (statusCode >= 400 && statusCode < 500) { + return true; + } + return false; + } } diff --git a/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy b/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000..b16dfe0 --- /dev/null +++ b/amazon-personalize-ranking/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,15 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant { + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + permission java.net.SocketPermission "*", "connect,resolve"; + permission java.lang.RuntimePermission "getClassLoader"; +}; diff --git a/src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java similarity index 85% rename from src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java index 63ebfdc..f72f947 100644 --- a/src/test/java/org/opensearch/search/relevance/SearchRelevancePluginIT.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/AmazonPersonalizeRankingPluginIT.java @@ -15,7 +15,7 @@ import java.io.IOException; -public class SearchRelevancePluginIT extends OpenSearchRestTestCase { +public class AmazonPersonalizeRankingPluginIT extends OpenSearchRestTestCase { public void testPluginInstalled() throws IOException, ParseException { Response response = client().performRequest(new Request("GET", "/_cat/plugins")); @@ -23,6 +23,6 @@ public void testPluginInstalled() throws IOException, ParseException { logger.info("response body: {}", body); assertNotNull(body); - assertTrue(body.contains("search-processor")); + assertTrue(body.contains("amazon-personalize-ranking")); } } diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java similarity index 56% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java index 71cbfdf..f83bbcb 100644 --- a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/PersonalizeRankingResponseProcessorTests.java @@ -19,6 +19,7 @@ import org.opensearch.env.TestEnvironment; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; +import org.opensearch.search.pipeline.Processor; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClient; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.client.PersonalizeClientSettings; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.requestparameter.PersonalizeRequestParameters; @@ -26,30 +27,34 @@ import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.SearchTestUtil; import org.opensearch.test.OpenSearchTestCase; -import java.util.Arrays; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.mockito.Mockito.mock; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.PersonalizeRankingResponseProcessor.TYPE; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; public class PersonalizeRankingResponseProcessorTests extends OpenSearchTestCase { - private static final String TYPE = PersonalizeRankingResponseProcessor.TYPE; - private Settings settings = buildEnvSettings(Settings.EMPTY); - private Environment env = TestEnvironment.newEnvironment(settings); - private String personalizeCampaign = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; - private String iamRoleArn = "arn:aws:iam::000000000000:role/test"; - private String itemIdField = "ITEM_ID"; - private String region = "us-west-2"; - private double weight = 1.0; - private int numHits = 10; + private static final Processor.PipelineContext UPDATE_CONTEXT = new Processor.PipelineContext(Processor.PipelineSource.UPDATE_PIPELINE); + private static final Processor.PipelineContext VALIDATE_CONTEXT = new Processor.PipelineContext(Processor.PipelineSource.VALIDATE_PIPELINE); + private final Settings settings = buildEnvSettings(Settings.EMPTY); + private final Environment env = TestEnvironment.newEnvironment(settings); + private static final String PERSONALIZE_CAMPAIGN = "arn:aws:personalize:us-west-2:000000000000:campaign/test-campaign"; + private static final String IAM_ROLE_ARN = "arn:aws:iam::000000000000:role/test"; + private static final String ITEM_ID_FIELD = "ITEM_ID"; + private static final String REGION = "us-west-2"; + private static final double WEIGHT = 1.0; + private static final int NUM_HITS = 10; - private PersonalizeClientSettings clientSettings = PersonalizeClientSettings.getClientSettings(env.settings()); + private final PersonalizeClientSettings clientSettings = PersonalizeClientSettings.getClientSettings(env.settings()); public void testCreateFactoryThrowsExceptionWithEmptyConfig() { PersonalizeRankingResponseProcessor.Factory factory @@ -60,21 +65,21 @@ public void testCreateFactoryThrowsExceptionWithEmptyConfig() { null, false, Collections.emptyMap(), - null + UPDATE_CONTEXT )); IdleConnectionReaper.shutdown(); } - public void testFactory() { + public void testFactoryValidations() { PersonalizeRankingResponseProcessor.Factory factory = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings); // Test config without campaign Map configuration = new HashMap<>(); - configuration.put("item_id_field", itemIdField); + configuration.put("item_id_field", ITEM_ID_FIELD); configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); - configuration.put("weight", String.valueOf(weight)); - configuration.put("iam_role_arn", iamRoleArn); - configuration.put("aws_region", region); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); expectThrows(OpenSearchParseException.class, () -> factory.create( Collections.emptyMap(), @@ -82,16 +87,16 @@ public void testFactory() { null, false, configuration, - null + VALIDATE_CONTEXT )); configuration.clear(); // Test config without recipe - configuration.put("campaign_arn", personalizeCampaign); - configuration.put("item_id_field", itemIdField); - configuration.put("weight", String.valueOf(weight)); - configuration.put("iam_role_arn", iamRoleArn); - configuration.put("aws_region", region); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); expectThrows(OpenSearchParseException.class, () -> factory.create( Collections.emptyMap(), @@ -99,59 +104,79 @@ public void testFactory() { null, false, configuration, - null + VALIDATE_CONTEXT )); configuration.clear(); // Test config without region - configuration.put("campaign_arn", personalizeCampaign); - configuration.put("item_id_field", itemIdField); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); - configuration.put("weight", String.valueOf(weight)); - configuration.put("iam_role_arn", iamRoleArn); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); expectThrows(OpenSearchParseException.class, () -> factory.create( Collections.emptyMap(), null, null, false, configuration, - null + VALIDATE_CONTEXT )); configuration.clear(); // Test config without weight - configuration.put("campaign_arn", personalizeCampaign); - configuration.put("item_id_field", itemIdField); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); - configuration.put("iam_role_arn", iamRoleArn); - configuration.put("aws_region", region); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); expectThrows(OpenSearchParseException.class, () -> factory.create( Collections.emptyMap(), null, null, false, configuration, - null + VALIDATE_CONTEXT )); configuration.clear(); // Test configuration with invalid weight value - configuration.put("campaign_arn", personalizeCampaign); - configuration.put("item_id_field", itemIdField); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); configuration.put("weight", "invalid"); - configuration.put("iam_role_arn", iamRoleArn); - configuration.put("aws_region", region); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); expectThrows(OpenSearchParseException.class, () -> factory.create( Collections.emptyMap(), null, null, false, configuration, - null + VALIDATE_CONTEXT )); configuration.clear(); - IdleConnectionReaper.shutdown(); + + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); + + // Test that we don't create client on validation + configuration.putAll(buildPersonalizeResponseProcessorConfig()); + PersonalizeRankingResponseProcessor processor = factory.create(Collections.emptyMap(), null, null, false, configuration, VALIDATE_CONTEXT); + assertNull(processor.getPersonalizeClient()); + + // Test that we fail on valid configuration in search request context + expectThrows(IllegalStateException.class, () -> factory.create( + Collections.emptyMap(), + null, + null, + false, + buildPersonalizeResponseProcessorConfig(), + new Processor.PipelineContext(Processor.PipelineSource.SEARCH_REQUEST))); } public void testCreateFactoryWithAllPersonalizeConfig() throws Exception { @@ -161,7 +186,7 @@ public void testCreateFactoryWithAllPersonalizeConfig() throws Exception { Map configuration = buildPersonalizeResponseProcessorConfig(); PersonalizeRankingResponseProcessor personalizeResponseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); assertEquals(TYPE, personalizeResponseProcessor.getType()); assertEquals("testTag", personalizeResponseProcessor.getTag()); @@ -177,7 +202,7 @@ public void testProcessorWithNoHits() throws Exception { Map configuration = buildPersonalizeResponseProcessorConfig(); PersonalizeRankingResponseProcessor personalizeResponseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); SearchRequest searchRequest = new SearchRequest(); SearchHits hits = new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0.0f); SearchResponseSections searchResponseSections = new SearchResponseSections(hits, null, null, false, false, null, 0); @@ -196,22 +221,22 @@ public void testProcessorWithPersonalizeContext() throws Exception { Map configuration = buildPersonalizeResponseProcessorConfig(); PersonalizeRankingResponseProcessor personalizeResponseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); Map personalizeContext = new HashMap<>(); personalizeContext.put("contextKey2", "contextValue2"); SearchResponse personalizedResponse = - getPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, numHits); + createPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, NUM_HITS); List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); List rerankedDocumentIds; rerankedDocumentIds = transformedHits.stream() - .filter(h -> h.getSourceAsMap().get(itemIdField) != null) - .map(h -> h.getSourceAsMap().get(itemIdField).toString()) + .filter(h -> h.getSourceAsMap().get(ITEM_ID_FIELD) != null) + .map(h -> h.getSourceAsMap().get(ITEM_ID_FIELD).toString()) .collect(Collectors.toList()); - ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numHits, 1); + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); IdleConnectionReaper.shutdown(); } @@ -224,13 +249,13 @@ public void testProcessorWithHitsWithInvalidPersonalizeContext() throws Exceptio Map configuration = buildPersonalizeResponseProcessorConfig(); PersonalizeRankingResponseProcessor personalizeResponseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); Map personalizeContext = new HashMap<>(); personalizeContext.put("contextKey2", 5); expectThrows(OpenSearchParseException.class, () -> - getPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, numHits)); + createPersonalizedRankingProcessorResponse(personalizeResponseProcessor, personalizeContext, NUM_HITS)); IdleConnectionReaper.shutdown(); } @@ -244,9 +269,9 @@ public void testPersonalizeRankingResponse() throws Exception { Map configuration = buildPersonalizeResponseProcessorConfig(); PersonalizeRankingResponseProcessor responseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); - SearchResponse personalizedResponse = getPersonalizedRankingProcessorResponse(responseProcessor, null, numHits); + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); List rerankedDocumentIds; @@ -255,7 +280,82 @@ public void testPersonalizeRankingResponse() throws Exception { .map(h -> h.getSourceAsMap().get(itemField).toString()) .collect(Collectors.toList()); - ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numHits, 1); + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2Response() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemField = "ITEM_ID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .filter(h -> h.getSourceAsMap().get(itemField) != null) + .map(h -> h.getSourceAsMap().get(itemField).toString()) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2ResponseWithInvalidItemIdFieldName() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemFieldInvalid = "ITEM_ID_NOT_VALID"; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + configuration.put("item_id_field", itemFieldInvalid); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + expectThrows(OpenSearchParseException.class, () -> + createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS)); + IdleConnectionReaper.shutdown(); + } + + public void testPersonalizeRankingV2ResponseWithDefaultItemIdField() throws Exception { + PersonalizeClient personalizeClient = PersonalizeRuntimeTestUtil.buildMockPersonalizeClient(); + + PersonalizeRankingResponseProcessor.Factory factory + = new PersonalizeRankingResponseProcessor.Factory(this.clientSettings, (cp, r) -> personalizeClient); + + String itemIdFieldEmpty = ""; + Map configuration = buildPersonalizeResponseProcessorConfig(); + configuration.put("item_id_field", itemIdFieldEmpty); + configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME); + + PersonalizeRankingResponseProcessor responseProcessor = + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); + + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); + + List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); + List rerankedDocumentIds; + rerankedDocumentIds = transformedHits.stream() + .map(SearchHit::getId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); + assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); IdleConnectionReaper.shutdown(); } @@ -271,10 +371,10 @@ public void testPersonalizeRankingResponseWithInvalidItemIdFieldName() throws Ex configuration.put("item_id_field", itemFieldInvalid); PersonalizeRankingResponseProcessor responseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); expectThrows(OpenSearchParseException.class, () -> - getPersonalizedRankingProcessorResponse(responseProcessor, null, numHits)); + createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS)); IdleConnectionReaper.shutdown(); } @@ -289,26 +389,26 @@ public void testPersonalizeRankingResponseWithDefaultItemIdField() throws Except configuration.put("item_id_field", itemIdFieldEmpty); PersonalizeRankingResponseProcessor responseProcessor = - factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, null); + factory.create(Collections.emptyMap(), "testTag", "testingAllFields", false, configuration, UPDATE_CONTEXT); - SearchResponse personalizedResponse = getPersonalizedRankingProcessorResponse(responseProcessor, null, numHits); + SearchResponse personalizedResponse = createPersonalizedRankingProcessorResponse(responseProcessor, null, NUM_HITS); List transformedHits = Arrays.asList(personalizedResponse.getHits().getHits()); List rerankedDocumentIds; rerankedDocumentIds = transformedHits.stream() - .filter(h -> h.getId() != null) - .map(h -> h.getId()) + .map(SearchHit::getId) + .filter(Objects::nonNull) .collect(Collectors.toList()); - ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(numHits, 1); + ArrayList expectedRankedDocumentIds = PersonalizeRuntimeTestUtil.expectedRankedItemIdsForGivenWeight(NUM_HITS, 1); assertEquals(expectedRankedDocumentIds, rerankedDocumentIds); IdleConnectionReaper.shutdown(); } - private SearchResponse getPersonalizedRankingProcessorResponse(PersonalizeRankingResponseProcessor responseProcessor, - Map personalizeContext, - int numHits) throws Exception { + private SearchResponse createPersonalizedRankingProcessorResponse(PersonalizeRankingResponseProcessor responseProcessor, + Map personalizeContext, + int numHits) throws Exception { PersonalizeRequestParameters personalizeRequestParams = new PersonalizeRequestParameters("user_1", personalizeContext); SearchRequest request = SearchTestUtil.createSearchRequestWithPersonalizeRequest(personalizeRequestParams); @@ -324,12 +424,12 @@ private SearchResponse getPersonalizedRankingProcessorResponse(PersonalizeRankin private Map buildPersonalizeResponseProcessorConfig() { Map configuration = new HashMap<>(); - configuration.put("campaign_arn", personalizeCampaign); - configuration.put("item_id_field", itemIdField); + configuration.put("campaign_arn", PERSONALIZE_CAMPAIGN); + configuration.put("item_id_field", ITEM_ID_FIELD); configuration.put("recipe", AMAZON_PERSONALIZED_RANKING_RECIPE_NAME); - configuration.put("weight", String.valueOf(weight)); - configuration.put("iam_role_arn", iamRoleArn); - configuration.put("aws_region", region); + configuration.put("weight", String.valueOf(WEIGHT)); + configuration.put("iam_role_arn", IAM_ROLE_ARN); + configuration.put("aws_region", REGION); return configuration; } } diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java similarity index 73% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java index a81564c..bf1def8 100644 --- a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientSettingsTests.java @@ -9,11 +9,17 @@ import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSSessionCredentials; +import org.opensearch.common.settings.SecureSetting; +import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.SettingsException; +import org.opensearch.core.common.settings.SecureString; import org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil.ACCESS_KEY; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.utils.PersonalizeClientSettingsTestUtil.SECRET_KEY; @@ -29,6 +35,17 @@ public void testWithBasicCredentials() throws IOException { assertFalse(credentials instanceof AWSSessionCredentials); } + public void testWithGetAllSetting() throws IOException { + PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); + assertEquals(clientSettings.getAllSettings().size(), 3); + Setting ACCESS_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.access_key", null); + Setting SECRET_KEY_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.secret_key", null); + Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("personalized_search_ranking.aws.session_token", null); + assertEquals(ACCESS_KEY_SETTING, clientSettings.getAllSettings().toArray()[0]); + assertEquals(SECRET_KEY_SETTING, clientSettings.getAllSettings().toArray()[1]); + assertEquals(SESSION_TOKEN_SETTING, clientSettings.getAllSettings().toArray()[2]); + } + public void testWithSessionCredentials() throws IOException { PersonalizeClientSettings clientSettings = PersonalizeClientSettingsTestUtil.buildClientSettings(true, true, true); AWSCredentials credentials = clientSettings.getCredentials(); diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeClientTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/client/PersonalizeCredentialsProviderFactoryTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/configuration/PersonalizeIntelligentRankerConfigurationTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/PersonalizeRankerFactoryTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java similarity index 89% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java index 32b4a61..1eec5cc 100644 --- a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/ranker/impl/AmazonPersonalizeRankerImplTests.java @@ -8,6 +8,8 @@ package org.opensearch.search.relevance.transformer.personalizeintelligentranking.ranker.impl; +import com.amazonaws.AmazonServiceException; +import org.junit.Assert; import org.mockito.Mockito; import org.opensearch.OpenSearchParseException; import org.opensearch.search.SearchHit; @@ -303,4 +305,39 @@ public void testReRankWithWeightAsNeitherZeroOrOneWithNullItemIdField() throws I assertNotEquals(rerankedDocumentIdsWhenWeightIsOne, rerankedDocumentIds); assertNotEquals(rerankedDocumentIdsWhenWeightIsZero, rerankedDocumentIds); } + + public void testReRankWithaccessDeniedExceptionWithStatusCode400() throws IOException { + + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenThrow(buildErrorWithStatusCode(400)); + + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Assert.assertThrows(IllegalArgumentException.class, () -> ranker.rerank(responseHits, requestParameters)); + } + + public void testReRankWithaccessDeniedExceptionWithStatusCode500() throws IOException { + + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, recipe, itemIdField, region, weight); + PersonalizeClient client = Mockito.mock(PersonalizeClient.class); + Mockito.when(client.getPersonalizedRanking(any())).thenThrow(buildErrorWithStatusCode(500)); + + PersonalizeRequestParameters requestParameters = new PersonalizeRequestParameters(); + requestParameters.setUserId("28"); + SearchHits responseHits = SearchTestUtil.getSampleSearchHitsForPersonalize(numOfHits); + AmazonPersonalizedRankerImpl ranker = new AmazonPersonalizedRankerImpl(rankerConfig, client); + Assert.assertThrows(AmazonServiceException.class, () -> ranker.rerank(responseHits, requestParameters)); + } + + + private AmazonServiceException buildErrorWithStatusCode(int statusCode) { + AmazonServiceException amazonServiceException = new AmazonServiceException("Error"); + amazonServiceException.setStatusCode(statusCode); + return amazonServiceException; + } } diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParameterUtilTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/requestparameter/PersonalizeRequestParametersExtBuilderTests.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeClientSettingsTestUtil.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/PersonalizeRuntimeTestUtil.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java similarity index 100% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/SearchTestUtil.java diff --git a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java similarity index 91% rename from src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java rename to amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java index 537d78f..1227915 100644 --- a/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java +++ b/amazon-personalize-ranking/src/test/java/org/opensearch/search/relevance/transformer/personalizeintelligentranking/utils/ValidationUtilTests.java @@ -13,6 +13,7 @@ import org.opensearch.test.OpenSearchTestCase; import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_RECIPE_NAME; +import static org.opensearch.search.relevance.transformer.personalizeintelligentranking.configuration.Constants.AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME; public class ValidationUtilTests extends OpenSearchTestCase { @@ -30,6 +31,12 @@ public void testValidRankerConfig () { ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG); } + public void testValidRankerConfigPersonalizedRankingV2 () { + PersonalizeIntelligentRankerConfiguration rankerConfig = + new PersonalizeIntelligentRankerConfiguration(personalizeCampaign, iamRoleArn, AMAZON_PERSONALIZED_RANKING_V2_RECIPE_NAME, itemIdField, region, weight); + ValidationUtil.validatePersonalizeIntelligentRankerConfiguration(rankerConfig, TYPE, TAG); + } + public void testInvalidCampaignArn () { PersonalizeIntelligentRankerConfiguration rankerConfig = new PersonalizeIntelligentRankerConfiguration("invalid:campaign/test", iamRoleArn, AMAZON_PERSONALIZED_RANKING_RECIPE_NAME, itemIdField, region, weight); diff --git a/src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java b/amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java similarity index 77% rename from src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java rename to amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java index 1f0d7dd..43e0349 100644 --- a/src/yamlRestTest/java/org/opensearch/search/relevance/SearchRelevanceClientYamlTestSuiteIT.java +++ b/amazon-personalize-ranking/src/yamlRestTest/java/org/opensearch/search/relevance/AmazonPersonalizeRankingClientYamlTestSuiteIT.java @@ -13,9 +13,9 @@ import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase; -public class SearchRelevanceClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { +public class AmazonPersonalizeRankingClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase { - public SearchRelevanceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + public AmazonPersonalizeRankingClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { super(testCandidate); } diff --git a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml similarity index 76% rename from src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml rename to amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml index 0b4f613..daa9333 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/amazon-personalize-ranking/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml @@ -5,7 +5,7 @@ h: component - match: - $body: /^opensearch-search-processor-\d+.\d+.\d+.\d+\n$/ + $body: /^opensearch-amazon-personalize-ranking-\d+.\d+.\d+.\d+\n$/ - do: indices.create: diff --git a/build.gradle b/build.gradle index 9d69664..3647cd1 100644 --- a/build.gradle +++ b/build.gradle @@ -1,156 +1,8 @@ -import org.opensearch.gradle.test.RestIntegTestTask - -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.yaml-rest-test' -apply plugin: 'opensearch.pluginzip' -apply plugin: 'jacoco' - -group = 'org.opensearch' - -def pluginName = 'search-processor' -def pluginDescription = 'Make Opensearch results more relevant.' -def projectPath = 'org.opensearch' -def pathToPlugin = 'search.relevance' -def pluginClassName = 'SearchRelevancePlugin' - -publishing { - publications { - pluginZip(MavenPublication) { publication -> - pom { - name = pluginName - description = pluginDescription - licenses { - license { - name = "The Apache License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - } - } - developers { - developer { - name = "OpenSearch" - url = "https://github.com/opensearch-project/search-processor" - } - } - } - } +ext { + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + opensearch_version = System.getProperty("opensearch.version", "3.0.0") + plugin_version = opensearch_version + if (isSnapshot) { + opensearch_version += "-SNAPSHOT" } } -opensearchplugin { - name "opensearch-${pluginName}-${plugin_version}.0" - description pluginDescription - classname "${projectPath}.${pathToPlugin}.${pluginClassName}" - licenseFile rootProject.file('LICENSE') - noticeFile rootProject.file('NOTICE') -} - -// This requires an additional Jar not published as part of build-tools -loggerUsageCheck.enabled = false - -// No need to validate pom, as we do not upload to maven/sonatype -validateNebulaPom.enabled = false - -buildscript { - ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") - opensearch_version = System.getProperty("opensearch.version", "3.0.0") - plugin_version = opensearch_version - if (isSnapshot) { - opensearch_version += "-SNAPSHOT" - } - } - - repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } - } - - dependencies { - classpath "org.opensearch.gradle:build-tools:${opensearch_version}" - } -} - -repositories { - mavenLocal() - maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } - mavenCentral() - maven { url "https://plugins.gradle.org/m2/" } -} - -dependencies { - implementation 'com.ibm.icu:icu4j:57.2' - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.httpcomponents:httpclient:4.5.14' - implementation 'org.apache.httpcomponents:httpcore:4.4.16' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.0' - implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.0' - implementation 'commons-logging:commons-logging:1.2' - implementation 'com.amazonaws:aws-java-sdk-sts:1.12.300' - implementation 'com.amazonaws:aws-java-sdk-core:1.12.300' - implementation 'com.amazonaws:aws-java-sdk-personalizeruntime:1.12.300' -} - - -allprojects { - plugins.withId('jacoco') { - jacoco.toolVersion = '0.8.9' - } -} - - -test { - include '**/*Tests.class' - finalizedBy jacocoTestReport -} - -task integTest(type: RestIntegTestTask) { - description = "Run tests against a cluster" - testClassesDirs = sourceSets.test.output.classesDirs - classpath = sourceSets.test.runtimeClasspath -} -tasks.named("check").configure { dependsOn(integTest) } - -integTest { - // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable - if (System.getProperty("test.debug") != null) { - jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' - } -} - -testClusters.integTest { - testDistribution = "INTEG_TEST" - - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) -} - -run { - useCluster testClusters.integTest -} - -sourceSets { - main { - resources { - srcDirs = ["config"] - includes = ["**/*.yml"] - } - } -} - - -jacocoTestReport { - dependsOn test - reports { - xml.required = true - html.required = true - } -} - -// TODO: Enable these checks -dependencyLicenses.enabled = false -thirdPartyAudit.enabled = false -loggerUsageCheck.enabled = false diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a7..a4b76b9 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 8196ca1..e1b837a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=a62c5f99585dd9e1f95dab7b9415a0e698fa9dd1e6c38537faa81ac078f4d23e -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-bin.zip +distributionSha256Sum=7a00d51fb93147819aab76024feece20b6b84e420694101f276be952e08bef03 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index aeb74cb..f5feea6 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -83,7 +85,9 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -130,10 +134,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -141,7 +148,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -149,7 +156,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -198,11 +205,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/gradlew.bat b/gradlew.bat index 6689b85..9b42019 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -43,11 +45,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail diff --git a/helpers/personalized_search_ranking_quickstart.sh b/helpers/personalized_search_ranking_quickstart.sh new file mode 100755 index 0000000..c78b3ad --- /dev/null +++ b/helpers/personalized_search_ranking_quickstart.sh @@ -0,0 +1,414 @@ +#!/bin/bash + +set -o errexit +set -o errtrace +set -o pipefail +set -o nounset + +# Some useful constants +readonly DOCKER_IMAGE_TAG="opensearch-with-personalized-search-ranking" +readonly OPENSEARCH_VERSION="2.9.0" + +# +# Set default values for OpenSearch (+Dashboards) image tags and plugin URL. +# We can pass them from outside to override these settings for testing purposes. +# +if [ -z "${OPENSEARCH_IMAGE_TAG:-}" ]; then + OPENSEARCH_IMAGE_TAG="opensearchproject/opensearch:${OPENSEARCH_VERSION}" +fi +if [ -z "${OPENSEARCH_DASHBOARDS_IMAGE_TAG:-}" ]; then + OPENSEARCH_DASHBOARDS_IMAGE_TAG="opensearchproject/opensearch-dashboards:${OPENSEARCH_VERSION}" +fi +if [ -z "${SEARCH_PROCESSOR_PLUGIN_URL:-}" ]; then + SEARCH_PROCESSOR_PLUGIN_URL="https://github.com/opensearch-project/search-processor/releases/download/${OPENSEARCH_VERSION}/opensearch-search-processor-${OPENSEARCH_VERSION}.0.zip" +fi + +function print_help() { + cat << EOF +Usage: $0 [-r ] [--profile ] + [--volume-name ] [--admin-password ] + -r | --region The AWS region for the Personalize Intelligent Ranking + service endpoint. If not specified, will read from the + AWS CLI for the default profile. + --profile The AWS profile to use for credentials. If not set, then + the script will try first to use credentials from the + environment, then from the default AWS profile. + --volume-name Without this option, the OpenSearch container will write + the index to ephemeral container storage, which is lost when + the container is removed. Using this option will map the + named Docker volume to \$OPENSEARCH_ROOT/data, so index data + will persist across executions. If the named volume does not + exist, it will be created. + --admin-password For OpenSearch 2.12 and higher, we no longer use a default + password of "admin" for the admin user. Instead, the value + passed to this parameter will be used as the admin password. + For OpenSearch versions prior to 2.12, this argument will be + ignored with a warning. + + NOTE: If the --profile option is not specified, the script will attempt to read AWS + credentials (access/secret key, optional session token) from environment variables, + and then from the default AWS profile, in order to pass them to the OpenSearch keystore + to be used to connect to the Personalize Intelligent ranking service. + If no credentials are found, the script WILL NOT pass credentials to the OpenSearch + keystore. When running a reranking request, the ranking plugin may rely on + instance profile credentials delivered through the EC2 metadata service, or credentials + from the ECS metadata service. +EOF +} + +# +# Parse and validate arguments +# + +while [ "$#" -gt 0 ]; do + case $1 in + -r | --region ) + shift + AWS_REGION=$1 + shift + ;; + -h | --help ) + print_help + exit 0 + ;; + --profile ) + shift + AWS_PROFILE=$1 + shift + ;; + --volume-name ) + shift + VOLUME_NAME=$1 + shift + ;; + --admin-password ) + shift + OPENSEARCH_INITIAL_ADMIN_PASSWORD="$1" + shift + ;; + esac +done + +# Starting in 2.12.0, security demo configuration script requires an initial admin password +OPENSEARCH_REQUIRED_VERSION="2.12.0" +COMPARE_VERSION=`echo $OPENSEARCH_REQUIRED_VERSION $OPENSEARCH_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` +if [ "$COMPARE_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then + if [ -n "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "WARNING: The --admin-password setting has no effect on OpenSearch ${OPENSEARCH_VERSION}. The admin password will be 'admin'." + fi + OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin" +elif [ -z "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "Starting with OpenSearch 2.12, you must specify the admin password with the --admin-password parameter." + exit 1 +fi + +# +# Determine which credentials and region to use. By the end of this block, all specified +# credentials will be loaded into environment variables (or we fail with an explanatory +# error message). +# +if [ -n "${AWS_PROFILE:-}" ]; then + # Load everything from the specified profile + AWS_ACCESS_KEY_ID=$(aws --profile ${AWS_PROFILE} configure get aws_access_key_id || echo) + AWS_SECRET_ACCESS_KEY=$(aws --profile ${AWS_PROFILE} configure get aws_secret_access_key || echo) + AWS_SESSION_TOKEN=$(aws --profile ${AWS_PROFILE} configure get aws_session_token || echo) + if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + >&2 echo "Unable to load credentials from profile ${AWS_PROFILE}" + exit 1 + elif [ -z "${AWS_SESSION_TOKEN}" ]; then + echo "Using AWS credentials (aws_access_key_id and aws_secret_access_key) from profile ${AWS_PROFILE}" + else + echo "Using AWS credentials (aws_access_key_id, aws_secret_access_key, and aws_session_token) from profile ${AWS_PROFILE}" + fi + if [ -z "${AWS_REGION:-}" ]; then + AWS_REGION=$(aws --profile ${AWS_PROFILE} configure get region || echo) + if [ -n "${AWS_REGION:-}" ]; then + echo "Using AWS region ${AWS_REGION} from profile ${AWS_PROFILE}" + else + >&2 echo "Argument [-r | --region] not specified and unable to infer region from profile ${AWS_PROFILE}" + exit 1 + fi + fi +else + # No profile set + if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then + if [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + echo "No profile set and no credentials in environment. Trying to load default profile credentials." + AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id || echo) + AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key || echo) + AWS_SESSION_TOKEN=$(aws configure get aws_session_token || echo) + if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY}" ]; then + echo "Unable to load credentials from default profile. No credentials will be passed to the OpenSearch keystore." + echo "OpenSearch will use the default credential provider chain to access Personalize, which may rely on EC2 instance" + echo "profile credentials or credentials from ECS metadata service." + elif [ -z "${AWS_SESSION_TOKEN}" ]; then + echo "Using AWS credentials (aws_access_key_id and aws_secret_access_key) from default profile." + else + echo "Using AWS credentials (aws_access_key_id, aws_secret_access_key, and aws_session_token) from default profile." + fi + else + >&2 echo "Environment variable AWS_SECRET_ACCESS_KEY is specified, but AWS_ACCESS_KEY_ID is not." + >&2 echo "Unable to determine which credentials to use." + exit 1 + fi + else + # AWS_ACCCESS_KEY_ID is set + if [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + >&2 echo "Environment variable AWS_ACCESS_KEY_ID is specified, but AWS_SECRET_ACCESS_KEY is not." + >&2 echo "Unable to determine which credentials to use." + exit 1 + else + if [ -n "${AWS_SESSION_TOKEN:-}" ]; then + echo "Using credentials from environment (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN)." + else + echo "Using credentials from environment (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)." + fi + fi + fi + if [ -z "${AWS_REGION:-}" ]; then + AWS_REGION=$(aws configure get region || echo) + if [ -n "${AWS_REGION:-}" ]; then + echo "Using AWS region ${AWS_REGION} from default profile" + else + >&2 echo "Argument [-r | --region] not specified and unable to infer region from default profile" + exit 1 + fi + fi +fi + +echo "Established AWS key id, secret key, session token and region." + +# +# Create a unique directory to hold the Dockerfile and docker-compose.yml files. +# +PLATFORM=$(uname) +if [ "${PLATFORM}" == "Darwin" ]; then + DOCKER_BUILD_DIR=$(mktemp -d opensearch-personalize-intelligent-ranking-docker.XXXX) +else + # Assume GNU mktemp + DOCKER_BUILD_DIR=$(mktemp -d -p . opensearch-personalize-intelligent-ranking-docker.XXXX) +fi + +cd ${DOCKER_BUILD_DIR} + +SUFFIX=$(echo ${DOCKER_BUILD_DIR} | sed s/.*\.//) +echo $SUFFIX + +echo "Running in $(pwd)" + +# +# Construct a Dockerfile that installs the search-processor plugin in the target image +# +cat >Dockerfile <>personalized_search_ranking.credentials <>personalized_search_ranking.credentials <install_credentials.sh <<"EOF" +#!/bin/bash + +for l in $(cat $1); do + KEY=$(echo $l | cut -f1 -d:) + VALUE=$(echo $l | cut -f2 -d:) + echo $VALUE | /usr/share/opensearch/bin/opensearch-keystore add $KEY --stdin +done +EOF + chmod 755 install_credentials.sh + cat >>Dockerfile << EOF + +# Push credentials to keystore +COPY --chown=opensearch:opensearch install_credentials.sh /tmp +COPY --chown=opensearch:opensearch personalized_search_ranking.credentials /tmp +RUN /usr/share/opensearch/bin/opensearch-keystore create +RUN --mount=type=secret,id=credentials,target=/tmp/personalized_search_ranking.credentials,required=true,mode=0444 \ + /tmp/install_credentials.sh /tmp/personalized_search_ranking.credentials +EOF +fi +echo "Opensearch credentials saved" +# +# Build and tag the Docker image with the plugin (and maybe credentials in the keystore) +# +if [ -f personalized_search_ranking.credentials ]; then + DOCKER_BUILDKIT=1 docker build --tag ${DOCKER_IMAGE_TAG} --secret id=credentials,src=personalized_search_ranking.credentials . + rm personalized_search_ranking.credentials +else + docker build --tag ${DOCKER_IMAGE_TAG} . +fi +echo "Docker image built and tagged with credentials" +# +# Make sure we have opensearch-dashboards: +# +docker pull ${OPENSEARCH_DASHBOARDS_IMAGE_TAG} +echo "Docker image pulled" + +if [ -n "${VOLUME_NAME:-}" ]; then + if ! docker volume inspect ${VOLUME_NAME}> /dev/null; then + echo "Creating volume ${VOLUME_NAME}"; + docker volume create ${VOLUME_NAME} + fi + DATA_DIR_BLOCK=" volumes: + - ${VOLUME_NAME}:/usr/share/opensearch/data" + VOLUME_BLOCK="volumes: + ${VOLUME_NAME}: + external: true" +fi +echo "Volume created" + + + +# +# Create a docker-compose.yml file that will launch an OpenSearch node with the image we +# just built and an OpenSearch Dashboards node that points to the OpenSearch node. +# +cat >docker-compose.yml <cleanup_resources.sh <README <" https://localhost:9200/ + +Index some data on OpenSearch by following instructions at +https://opensearch.org/docs/latest/opensearch/index-data/ + + +Connect to OpenSearch Dashboards with a web browser at http://localhost:5601/, +using username admin and password admin. Select "Search Relevance" from the +top-left menu. In the resulting UI, you can submit a query without Personalized +search ranking and one with Personalized search Ranking. + +To configure and setup Personalize search ranking, run a curl command as follows: + +curl -X PUT "https://localhost:9200/_search/pipeline/intelligent_ranking" -u 'admin:' --insecure -H 'Content-Type: application/json' -d' +{ + "description": "A pipeline to apply custom reranking", + "response_processors" : [ + { + "personalized_search_ranking" : { + "campaign_arn" : "", + "item_id_field" : "", + "recipe" : "", + "weight" : "", + "iam_role_arn": "", + "aws_region": "" + } + } + ] +}' + +Interact with the Docker containers using docker-compose from directory + $(pwd) + +Some helpful docker-compose commands: + + docker-compose logs opensearch-node + Outputs latest logs from the OpenSearch server + + docker-compose logs opensearch-dashboard + Outputs latest logs from the OpenSearch Dashboard server + + docker-compose down + Shut down and clean up both containers. + + docker-compose up -d + Bring both containers back up. + +You can clean up all Docker containers and any execution plans created (if +applicable) by running + $(pwd)/cleanup_resources.sh + +The full text of this message is also available at + $(pwd)/README +EOF +cat README diff --git a/helpers/search_processing_kendra_quickstart.sh b/helpers/search_processing_kendra_quickstart.sh index 5351e5a..3c0bba4 100755 --- a/helpers/search_processing_kendra_quickstart.sh +++ b/helpers/search_processing_kendra_quickstart.sh @@ -27,7 +27,7 @@ function print_help() { cat << EOF Usage: $0 [-p ] [-r ] [-e ] [--profile ] [--create-execution-plan] - [--volume-name ] + [--volume-name ] [--admin-password ] -p | --execution-plan-id The ID returned from Kendra Intelligent Ranking service from the call to CreateRescoreExecutionPlan. Required if --create-execution-plan is not set. @@ -50,6 +50,11 @@ Usage: $0 [-p ] [-r ] [-e ] named Docker volume to \$OPENSEARCH_ROOT/data, so index data will persist across executions. If the named volume does not exist, it will be created. + --admin-password For OpenSearch 2.12 and higher, we no longer use a default + password of "admin" for the admin user. Instead, the value + passed to this parameter will be used as the admin password. + For OpenSearch versions prior to 2.12, this argument will be + ignored with a warning. NOTE: If the --profile option is not specified, the script will attempt to read AWS credentials (access/secret key, optional session token) from environment variables, @@ -101,6 +106,11 @@ while [ "$#" -gt 0 ]; do VOLUME_NAME=$1 shift ;; + --admin-password ) + shift + OPENSEARCH_INITIAL_ADMIN_PASSWORD="$1" + shift + ;; esac done @@ -121,6 +131,19 @@ if [ "${FAILED_VALIDATION}" == "1" ]; then exit 1 fi +# Starting in 2.12.0, security demo configuration script requires an initial admin password +OPENSEARCH_REQUIRED_VERSION="2.12.0" +COMPARE_VERSION=`echo $OPENSEARCH_REQUIRED_VERSION $OPENSEARCH_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` +if [ "$COMPARE_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then + if [ -n "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "WARNING: The --admin-password setting has no effect on OpenSearch ${OPENSEARCH_VERSION}. The admin password will be 'admin'." + fi + OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin" +elif [ -z "${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-}" ]; then + echo "Starting with OpenSearch 2.12, you must specify the admin password with the --admin-password parameter." + exit 1 +fi + # # Determine which credentials and region to use. By the end of this block, all specified # credentials will be loaded into environment variables (or we fail with an explanatory @@ -379,6 +402,7 @@ services: - kendra_intelligent_ranking.service.endpoint=${KENDRA_RANKING_ENDPOINT} - kendra_intelligent_ranking.service.region=${AWS_REGION} - kendra_intelligent_ranking.service.execution_plan_id=${EXECUTION_PLAN_ID} + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} ulimits: memlock: soft: -1 @@ -446,8 +470,8 @@ cat >README <" https://localhost:9200/ Index some data on OpenSearch by following instructions at https://opensearch.org/docs/latest/opensearch/index-data/ diff --git a/settings.gradle b/settings.gradle index cb5e308..96ba866 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,3 +8,5 @@ */ rootProject.name = 'search-processor' +include 'amazon-kendra-intelligent-ranking' +include 'amazon-personalize-ranking'