setup:
  - do:
     indices.create:
         index: foo
         body:
           settings:
             index:
               number_of_shards: 1
  - do:
      index:
        index:   foo
        id:      doc1
        body:    { "text": "berlin" }

  - do:
      index:
        index:  foo
        id:     doc2
        body:   { "text": "amsterdam" }

  - do:
      index:
        index:  foo
        id:     doc3
        body:   { "text": "amsterdam" }

  - do:
      index:
        index:  foo
        id:     doc4
        body:   { "text": "something about amsterdam and berlin" }

  - do:
      indices.refresh: {}

  - do:
      indices.put_alias:
        index: foo
        name: alias

---
"Response format":
  - do:
      rank_eval:
        index: foo,
        search_type: query_then_fetch
        body: {
          "requests" : [
            {
                "id": "amsterdam_query",
                "request": { "query": { "match" : {"text" : "amsterdam" }}},
                "ratings": [
                    {"_index": "foo", "_id": "doc1", "rating": 0},
                    {"_index": "foo", "_id": "doc2", "rating": 1},
                    {"_index": "foo", "_id": "doc3", "rating": 1}]
            },
            {
                "id" : "berlin_query",
                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
                "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
            }
          ],
          "metric" : { "precision": { "ignore_unlabeled" : true }}
        }

  - match: { metric_score: 1}
  - match: { details.amsterdam_query.metric_score: 1.0}
  - match: { details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
  - match: { details.amsterdam_query.metric_details.precision: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}}

  - length: { details.amsterdam_query.hits: 3}
  - match: { details.amsterdam_query.hits.0.hit._id:  "doc2"}
  - match: { details.amsterdam_query.hits.0.rating: 1}
  - match: { details.amsterdam_query.hits.1.hit._id:  "doc3"}
  - match: { details.amsterdam_query.hits.1.rating: 1}
  - match: { details.amsterdam_query.hits.2.hit._id:  "doc4"}
  - is_false: details.amsterdam_query.hits.2.rating

  - match: { details.berlin_query.metric_score: 1.0}
  - match: { details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc4"}]}
  - match: { details.berlin_query.metric_details.precision: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
  - length: { details.berlin_query.hits: 2}
  - match: { details.berlin_query.hits.0.hit._id: "doc1" }
  - match: { details.berlin_query.hits.0.rating: 1}
  - match: { details.berlin_query.hits.1.hit._id: "doc4" }
  - is_false: details.berlin_query.hits.1.rating

---
"Alias resolution":

  - do:
      rank_eval:
        index: alias
        body: {
          "requests" : [
            {
                "id": "amsterdam_query",
                "request": { "query": { "match" : {"text" : "amsterdam" }}},
                "ratings": [
                    {"_index": "foo", "_id": "doc1", "rating": 0},
                    {"_index": "foo", "_id": "doc2", "rating": 1},
                    {"_index": "foo", "_id": "doc3", "rating": 1}]
            },
            {
                "id" : "berlin_query",
                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
                "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
            }
          ],
          "metric" : { "precision": { "ignore_unlabeled" : true }}
        }

  - match: { metric_score: 1}
  - match: { details.amsterdam_query.metric_score: 1.0}
  - match: { details.berlin_query.metric_score: 1.0}

---
"Mean Reciprocal Rank":
  - do:
      rank_eval:
        body: {
          "requests" : [
            {
                "id": "amsterdam_query",
                "request": { "query": { "match" : {"text" : "amsterdam" }}},
                # doc4 should be returned in third position, so reciprocal rank is 1/3
                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
            },
            {
                "id" : "berlin_query",
                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
                # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2
                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
            }
          ],
          "metric" : { "mean_reciprocal_rank": {} }
        }

# average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663
  - gt: {metric_score: 0.416}
  - lt: {metric_score: 0.417}
  - gt: {details.amsterdam_query.metric_score: 0.333}
  - lt: {details.amsterdam_query.metric_score: 0.334}
  - match: {details.amsterdam_query.metric_details.mean_reciprocal_rank: {"first_relevant": 3}}
  - match: {details.amsterdam_query.unrated_docs:  [ {"_index": "foo", "_id": "doc2"},
                                                               {"_index": "foo", "_id": "doc3"} ]}
  - match: {details.berlin_query.metric_score: 0.5}
  - match: {details.berlin_query.metric_details.mean_reciprocal_rank: {"first_relevant": 2}}
  - match: {details.berlin_query.unrated_docs:  [ {"_index": "foo", "_id": "doc1"}]}

---
"Expected Reciprocal Rank":
  - do:
      rank_eval:
        body: {
          "requests" : [
            {
                "id": "amsterdam_query",
                "request": { "query": { "match" : {"text" : "amsterdam" }}},
                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
            },
            {
                "id" : "berlin_query",
                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
                "ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
            }
          ],
          "metric" : {
              "expected_reciprocal_rank": {
                  "maximum_relevance" : 1,
                  "k" : 5
              }
          }
        }

  - gt: {metric_score: 0.2083333}
  - lt: {metric_score: 0.2083334}
  - match: {details.amsterdam_query.metric_details.expected_reciprocal_rank.unrated_docs: 2}
  - match: {details.berlin_query.metric_details.expected_reciprocal_rank.unrated_docs: 1}