/* SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ /* * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. * * Licensed to Elasticsearch B.V. under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch B.V. licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Runtime.Serialization; using OpenSearch.OpenSearch.Xunit.XunitPlumbing; using OpenSearch.Net; using FluentAssertions; using OpenSearch.Client; using OpenSearch.Client.JsonNetSerializer; using Newtonsoft.Json; using Tests.Core.Client; using Tests.Core.Client.Settings; using Tests.Core.Xunit; using Tests.Domain; using Tests.Framework; using static Tests.Core.Serialization.SerializationTestHelper; using static OpenSearch.Client.Infer; using Field = OpenSearch.Client.Field; // ReSharper disable ArrangeMethodOrOperatorBody namespace Tests.ClientConcepts.HighLevel.Inference { public class FieldInference { /**[[field-inference]] * === Field inference * * Several places in the OpenSearch API expect the path to a field from your original source document, as a string value. * OSC allows you to use C# expressions to strongly type these field path strings. * * These expressions are assigned to a type called `Field`, and there are several ways to create an instance of one */ /** * ==== Constructor * Using the constructor directly is possible _but_ can get rather involved when resolving from a member access lambda expression */ [U] public void UsingConstructors() { var fieldString = new Field("name"); var fieldProperty = new Field(typeof(Project).GetProperty(nameof(Project.Name))); Expression> expression = p => p.Name; var fieldExpression = new Field(expression); Expect("name") .WhenSerializing(fieldExpression) .WhenSerializing(fieldString) .WhenSerializing(fieldProperty); } [U] public void UsingConstructorAlsoSetsComparisonValue() { /** When using the constructor and passing a value for `Name`, `Property` or `Expression`, * `ComparisonValue` is also set on the `Field` instance; this is used when * * - determining `Field` equality * - getting the hash code for a `Field` instance */ var fieldStringWithBoostTwo = new Field("name^2"); var fieldStringWithBoostThree = new Field("name^3"); Expression> expression = p => p.Name; var fieldExpression = new Field(expression); var fieldProperty = new Field(typeof(Project).GetProperty(nameof(Project.Name))); fieldStringWithBoostTwo.GetHashCode().Should().NotBe(0); fieldStringWithBoostThree.GetHashCode().Should().NotBe(0); fieldExpression.GetHashCode().Should().NotBe(0); fieldProperty.GetHashCode().Should().NotBe(0); fieldStringWithBoostTwo.Should().Be(fieldStringWithBoostTwo); //<1> <> } /** * [[field-name-with-boost]] *==== Field Names with Boost * * When specifying a `Field` name, the name can include a boost value; OSC will split the name and boost * value and set the `Boost` property; a boost value as part of the string takes precedence over a boost * value that may also be passed as the second constructor argument */ [U] public void NameCanSpecifyBoost() { Field fieldString = "name^2"; var fieldStringConstructor = new Field("name^2"); var fieldStringCreate = new Field("name^2", 3); //<1> OSC will take the boost from the name fieldString.Name.Should().Be("name"); fieldStringConstructor.Name.Should().Be("name"); fieldStringCreate.Name.Should().Be("name"); fieldString.Boost.Should().Be(2); fieldStringConstructor.Boost.Should().Be(2); fieldStringCreate.Boost.Should().Be(2); } /** * ==== Implicit Conversion * As well as using the constructor, you can also implicitly convert `string`, `PropertyInfo` and member access lambda expressions to a `Field`. * For expressions however, this is _still_ rather involved as the expression first needs to be assigned to a variable that explicitly specifies * the expression delegate type. */ [U] public void ImplicitConversion() { Field fieldString = "name"; Field fieldProperty = typeof(Project).GetProperty(nameof(Project.Name)); Expression> expression = p => p.Name; Field fieldExpression = expression; Expect("name") .WhenSerializing(fieldString) .WhenSerializing(fieldProperty) .WhenSerializing(fieldExpression); } /** * [[osc-infer]] * ==== Using OpenSearch.Client.Infer methods * To ease creating a `Field` instance from expressions, there is a static `Infer` class you can use * * [TIP] * ==== * This example uses the https://msdn.microsoft.com/en-us/library/sf0df423.aspx#Anchor_0[static import] `using static OpenSearch.Client.Infer;` * in the using directives to shorthand `OpenSearch.Client.Infer.Field()` * to simply `Field()`. Be sure to include this static import if copying any of these examples. * ==== * */ [U] public void UsingStaticPropertyField() { Field fieldString = "name"; /** but for expressions this is still rather involved */ var fieldExpression = Field(p => p.Name); /** this can be even shortened even further using a static import. * Now that is much terser then our first example using the constructor! */ fieldExpression = Field(p => p.Name); Expect("name") .WhenSerializing(fieldString) .WhenSerializing(fieldExpression); /** You can specify boosts in the field using a string, as well as using `OpenSearch.Client.Infer.Field` */ fieldString = "name^2.1"; fieldString.Boost.Should().Be(2.1); fieldExpression = Field(p => p.Name, 2.1); fieldExpression.Boost.Should().Be(2.1); Expect("name^2.1") .WhenSerializing(fieldString) .WhenSerializing(fieldExpression); } /** * [[camel-casing]] * ==== Field name casing * By default, OSC https://en.wikipedia.org/wiki/Camel_case[camelcases] **all** field names to better align with typical * JavaScript and JSON conventions */ [U] public void DefaultFieldNameInferrer() { /** using `DefaultFieldNameInferrer()` on ConnectionSettings you can change this behavior */ var setup = WithConnectionSettings(s => s.DefaultFieldNameInferrer(p => p.ToUpper())); setup.Expect("NAME").WhenSerializing(Field(p => p.Name)); /** A `Field` constructed from a `string` however is *always* passed along verbatim */ setup.Expect("NaMe").WhenSerializing("NaMe"); /** If you'd like OSC to not change the casing of field names at all, * simply pass a Func to `DefaultFieldNameInferrer` that simply returns the * input string */ setup = WithConnectionSettings(s => s.DefaultFieldNameInferrer(p => p)); setup.Expect("Name").WhenSerializing(Field(p => p.Name)); } /** * ==== Complex field name expressions */ [U] public void ComplexFieldNameExpressions() { /** You can follow your property expression to any depth. Here we are traversing to the `LeadDeveloper` `FirstName` */ Expect("leadDeveloper.firstName").WhenSerializing(Field(p => p.LeadDeveloper.FirstName)); /** When dealing with collection indexers, the indexer access is ignored allowing you to traverse into properties of collections */ Expect("curatedTags").WhenSerializing(Field(p => p.CuratedTags[0])); /** Similarly, LINQ's `.First()` method also works */ Expect("curatedTags").WhenSerializing(Field(p => p.CuratedTags.First())); Expect("curatedTags.added").WhenSerializing(Field(p => p.CuratedTags[0].Added)); Expect("curatedTags.name").WhenSerializing(Field(p => p.CuratedTags.First().Name)); /** NOTE: Remember, these are _expressions_ to access members, and not actual code that will be executed * * An indexer on a dictionary is assumed to describe a property name */ Expect("metadata.hardcoded").WhenSerializing(Field(p => p.Metadata["hardcoded"])); Expect("metadata.hardcoded.created").WhenSerializing(Field(p => p.Metadata["hardcoded"].Created)); /** A cool feature here is that OSC will evaluate variables passed to an indexer */ var variable = "var"; Expect("metadata.var").WhenSerializing(Field(p => p.Metadata[variable])); Expect("metadata.var.created").WhenSerializing(Field(p => p.Metadata[variable].Created)); /** * If you are using OpenSearchearch's multi-fields, which you really should as they allow * you to analyze a string in a number of different ways, these __"virtual"__ sub fields * do not always map back on to your POCO. By calling `.Suffix()` on expressions, you describe the sub fields that * should be mapped and <> */ Expect("leadDeveloper.firstName.raw").WhenSerializing( Field(p => p.LeadDeveloper.FirstName.Suffix("raw"))); Expect("curatedTags.raw").WhenSerializing( Field(p => p.CuratedTags[0].Suffix("raw"))); Expect("curatedTags.raw").WhenSerializing( Field(p => p.CuratedTags.First().Suffix("raw"))); Expect("curatedTags.added.raw").WhenSerializing( Field(p => p.CuratedTags[0].Added.Suffix("raw"))); Expect("metadata.hardcoded.raw").WhenSerializing( Field(p => p.Metadata["hardcoded"].Suffix("raw"))); Expect("metadata.hardcoded.created.raw").WhenSerializing( Field(p => p.Metadata["hardcoded"].Created.Suffix("raw"))); /** * You can even chain `.Suffix()` calls to any depth! */ Expect("curatedTags.name.raw.evendeeper").WhenSerializing( Field(p => p.CuratedTags.First().Name.Suffix("raw").Suffix("evendeeper"))); /** Variables passed to suffix will be evaluated as well */ var suffix = "unanalyzed"; Expect("metadata.var.unanalyzed").WhenSerializing( Field(p => p.Metadata[variable].Suffix(suffix))); Expect("metadata.var.created.unanalyzed").WhenSerializing( Field(p => p.Metadata[variable].Created.Suffix(suffix))); } /** * Suffixes can also be appended to expressions using `.AppendSuffix()`. This is useful in cases where you want to apply the same suffix * to a list of fields. */ [U] public void AppendingSuffixToExpressions() { /** Here we have a list of expressions */ var expressions = new List>> { p => p.Name, p => p.Description, p => p.CuratedTags.First().Name, p => p.LeadDeveloper.FirstName, p => p.Metadata["hardcoded"] }; /** and we want to append the suffix "raw" to each */ var fieldExpressions = expressions.Select>, Field>(e => e.AppendSuffix("raw")).ToList(); Expect("name.raw").WhenSerializing(fieldExpressions[0]); Expect("description.raw").WhenSerializing(fieldExpressions[1]); Expect("curatedTags.name.raw").WhenSerializing(fieldExpressions[2]); Expect("leadDeveloper.firstName.raw").WhenSerializing(fieldExpressions[3]); Expect("metadata.hardcoded.raw").WhenSerializing(fieldExpressions[4]); /** or we might even want to chain multiple `.AppendSuffix()` calls */ var multiSuffixFieldExpressions = expressions.Select>, Field>(e => e.AppendSuffix("raw").AppendSuffix("evendeeper")).ToList(); Expect("name.raw.evendeeper").WhenSerializing(multiSuffixFieldExpressions[0]); Expect("description.raw.evendeeper").WhenSerializing(multiSuffixFieldExpressions[1]); Expect("curatedTags.name.raw.evendeeper").WhenSerializing(multiSuffixFieldExpressions[2]); Expect("leadDeveloper.firstName.raw.evendeeper").WhenSerializing(multiSuffixFieldExpressions[3]); Expect("metadata.hardcoded.raw.evendeeper").WhenSerializing(multiSuffixFieldExpressions[4]); } /** * [[member-expressions-only]] * ==== Member Expressions only * The expression passed to Field should only be a MemberExpression * https://docs.microsoft.com/en-us/dotnet/api/system.linq.expressions.memberexpression?view=netframework-4.7.2 * */ [U] public void OnlyMemberExpressionAllowed() { var fieldExpression = Field(p => p.Name + 2); } /**[[field-name-attribute]] * ==== Attribute based naming * * Using OSC's property attributes you can specify a new name for the properties */ public class BuiltIn { [Text(Name = "naam")] public string Name { get; set; } } [U] public void BuiltInAnnotiatons() { Expect("naam").WhenSerializing(Field(p => p.Name)); } /**[[data-member-field-attribute]] * ==== DataMember attributes * * If a property has a `System.Runtime.Serialization.DataMemberAttribute` applied, this can be used to resolve * a field value for a property */ public class DataMember { [DataMember(Name = "nameFromDataMember")] public string Name { get; set; } } [U] public void DataAnnotationAttribute() { Expect("nameFromDataMember").WhenSerializing(Field(p => p.Name)); } /**[[serializer-specific-field-attribute]] * ==== Serializer specific attributes * * OSC can also use a serializer specific attribute to resolve a field value for a property. * In this example, the {nuget}/OpenSearch.Client.JsonNetSerializer[`JsonNetSerializer`] is hooked up as the * <> for the client and we use the `JsonPropertyAttribute` to resolve a field */ public class SerializerSpecific { [PropertyName("nameInJson"), JsonProperty("nameInJson")] public string Name { get; set; } } [U] public void SerializerSpecificAnnotations() { Expect("nameInJson").WhenSerializing(Field(p => p.Name)); } /** * If both a OSC property attribute and a serializer specific attribute are present on a property, * **OSC attributes take precedence** */ public class Both { [Text(Name = "naam")] [PropertyName("nameInJson"), DataMember(Name = "nameInJson")] public string Name { get; set; } } [U] public void OscAttributeTakesPrecedence() { Expect("naam").WhenSerializing(Field(p => p.Name)); Expect(new { naam = "Martijn Laarman" }).WhenSerializing(new Both { Name = "Martijn Laarman" }); } /** * [[field-inference-caching]] *==== Field Inference Caching * * Resolution of field names is cached _per_ `ConnectionSettings` instance. To demonstrate, * take the following simple POCOs */ class A { public C C { get; set; } } class B { public C C { get; set; } } class C { public string Name { get; set; } } [U] public void ExpressionsAreCachedButSeeDifferentTypes() { var client = TestClient.Default; var fieldNameOnA = client.Infer.Field(Field(p => p.C.Name)); var fieldNameOnB = client.Infer.Field(Field(p => p.C.Name)); /** * Here we have two similarly shaped expressions, one coming from A and one from B * that will resolve to the same field name, as expected */ fieldNameOnA.Should().Be("c.name"); fieldNameOnB.Should().Be("c.name"); /** * now we create a new connection settings with a re-map for `C` on class `A` to `"d"` * now when we resolve the field path for property `C` on `A`, it will be different than * for property `C` on `B` */ var newConnectionSettings = new TestConnectionSettings() .DefaultMappingFor(m => m .PropertyName(p => p.C, "d") ); var newClient = new OpenSearchClient(newConnectionSettings); fieldNameOnA = newClient.Infer.Field(Field(p => p.C.Name)); fieldNameOnB = newClient.Infer.Field(Field(p => p.C.Name)); fieldNameOnA.Should().Be("d.name"); fieldNameOnB.Should().Be("c.name"); /** however we didn't break inference on the first client instance using its separate connection settings */ fieldNameOnA = client.Infer.Field(Field(p => p.C.Name)); fieldNameOnB = client.Infer.Field(Field(p => p.C.Name)); fieldNameOnA.Should().Be("c.name"); fieldNameOnB.Should().Be("c.name"); } /** * [[field-inference-precedence]] * ==== Inference Precedence * To wrap up, the precedence in which field names are inferred is: * * . A naming of the property on `ConnectionSettings` using `.PropertyName()` * . A OSC `PropertyNameAttribute` * . Ask the serializer if the property has a verbatim value, e.g. it has a `JsonPropertyAttribute` if using {nuget}/OpenSearch.Client.JsonNetSerializer[`JsonNetSerializer`] * . See if the `MemberInfo` has a `DataMemberAttribute` applied * . Pass the `MemberInfo` to the `DefaultFieldNameInferrer`, which by default will camel case the `Name` property * * The following example class will demonstrate this precedence */ private class Precedence { [Text(Name = "renamedIgnoresOsc")] [PropertyName("renamedIgnoresJsonProperty"),JsonProperty("renamedIgnoresJsonProperty")] public string RenamedOnConnectionSettings { get; set; } //<1> Even though this property has various attributes applied we provide an override on ConnectionSettings later that takes precedence. [Text(Name = "oscAtt")] [PropertyName("oscProp"),JsonProperty("jsonProp")] public string OscAttribute { get; set; } //<2> Has a `TextAttribute`, `PropertyNameAttribute` and a `JsonPropertyAttribute` - the `TextAttribute` takes precedence. [PropertyName("oscProp"),JsonProperty("jsonProp")] public string OscProperty { get; set; } //<3> Has both a `PropertyNameAttribute` and a `JsonPropertyAttribute` - the `PropertyNameAttribute` takes precedence. [DataMember(Name ="jsonProp")] public string JsonProperty { get; set; } //<4> `JsonPropertyAttribute` takes precedence. [PropertyName("dontaskme"),JsonProperty("dontaskme")] public string AskSerializer { get; set; } //<5> This property we are going to hard code in our custom serializer to resolve to ask. [DataMember(Name = "data")] public string DataMember { get; set; } public string DefaultFieldNameInferrer { get; set; } //<6> We are going to register a DefaultFieldNameInferrer on ConnectionSettings that will uppercase all properties. } /** * We'll create a custom `IPropertyMappingProvider` that renames any property named `AskSerializer` to `ask`. * and hook it up when creating the Connection Settings in the following section. */ private class CustomPropertyMappingProvider : PropertyMappingProvider { public override IPropertyMapping CreatePropertyMapping(MemberInfo memberInfo) { return memberInfo.Name == nameof(Precedence.AskSerializer) ? new PropertyMapping { Name = "ask" } : base.CreatePropertyMapping(memberInfo); } } [U] [ProjectReferenceOnly] public void PrecedenceIsAsExpected() { /** Now, when we create the Connection Settings to use to configure the client, we'll add * - a default mapping for the `Precedence` type * - our `CustomPropertyMappingProvider` * - a delegate to perform default field name inference */ var usingSettings = WithConnectionSettings(s => s .DefaultMappingFor(m => m .PropertyName(p => p.RenamedOnConnectionSettings, "renamed") // <1> Rename on the mapping for the `Precedence` type ) .DefaultFieldNameInferrer(p => p.ToUpperInvariant()) // <2> Default inference for a field, if no other rules apply or are specified for a given field ).WithPropertyMappingProvider(new CustomPropertyMappingProvider()); // <3> Hook up the custom `IPropertyMappingProvider` usingSettings.Expect("renamed").ForField(Field(p => p.RenamedOnConnectionSettings)); usingSettings.Expect("oscAtt").ForField(Field(p => p.OscAttribute)); usingSettings.Expect("oscProp").ForField(Field(p => p.OscProperty)); usingSettings.Expect("jsonProp").ForField(Field(p => p.JsonProperty)); usingSettings.Expect("ask").ForField(Field(p => p.AskSerializer)); usingSettings.Expect("data").ForField(Field(p => p.DataMember)); usingSettings.Expect("DEFAULTFIELDNAMEINFERRER").ForField(Field(p => p.DefaultFieldNameInferrer)); /** The same naming rules also apply when indexing a document */ usingSettings.Expect(new [] { "ask", "DEFAULTFIELDNAMEINFERRER", "jsonProp", "oscProp", "oscAtt", "renamed", "data" }).AsPropertiesOf(new Precedence { RenamedOnConnectionSettings = "renamed on connection settings", OscAttribute = "using a osc attribute", OscProperty = "using a osc property", JsonProperty = "the default serializer resolves json property attributes", AskSerializer = "serializer fiddled with this one", DefaultFieldNameInferrer = "shouting much?", DataMember = "using a DataMember attribute" }); } /** *[[inherited-field-inference]] * ==== Overriding inherited field inference * * Properties inherited from a base type can be ignored and renamed using `DefaultMappingFor` for * a given type, on Connection Settings. * * To demonstrate, the `IgnoreMe` property on `Parent` can be ignored on the `Child` type, and the * `Description` property renamed, using `DefaultMappingFor(...)` */ public class Parent { public int Id { get; set; } public string Description { get; set; } public string IgnoreMe { get; set; } } public class Child : Parent { } [U] public void CodeBasedConfigurationInherits() { var usingSettings = WithConnectionSettings(s => s .DefaultMappingFor(m => m .PropertyName(p => p.Description, "desc") .Ignore(p => p.IgnoreMe) ) ); usingSettings.Expect(new [] { "id", "desc", }).AsPropertiesOf(new Child { Id = 1, Description = "this property will be renamed for Child", IgnoreMe = "this property will be ignored (won't be serialized) for Child", }); } public class SourceModel { [PropertyName("gexo")] public GeoModel Geo { get; set; } } public class GeoModel { [DataMember(Name = "country_iso_code")] public string CountryIsoCode { get; set; } } /// this tests runs both during development as wel as during canary builds which reference /// all the artifacts as nuget packages instead of project references [U] public void JsonPropertyIsPickedWhenSerializerIsInternalized() { var usingSettings = WithConnectionSettings(s => s) .WithSourceSerializer(JsonNetSerializer.Default); usingSettings.Expect("gexo").ForField(Field(p=>p.Geo)); usingSettings.Expect("country_iso_code").ForField(Field(p=>p.CountryIsoCode)); usingSettings.Expect(new [] { "country_iso_code", }).AsPropertiesOf(new GeoModel { CountryIsoCode = "nl" }); usingSettings.Expect(new [] { "gexo", }).AsPropertiesOf(new SourceModel { Geo = new GeoModel { CountryIsoCode = "nl" } }); usingSettings.Expect("gexo.country_iso_code").ForField(Field(p=>p.Geo.CountryIsoCode)); } } }