Rawnet senior PHP Developer Mark Bennett looks at Elasticsearch, and the benefits of putting it on your website. of the API and NodeJS www.rawnet.com

1. Elasticsearch

2. What Is Elasticsearch?
Search index
Scalable
Distributed
Document-oriented
Built on Apache Lucene

3. What Isn’t Elasticsearch
Not a database
No security
No ACID compliance
No support for transactions
Near real-time data availability
Eventual consistency

4. Typical Architecture
Web application
Data access layer
RDBMS Elasticsearch

5. Who Uses Elasticsearch?

6. Simple REST API - GET
GET / {
"name": "Murmur",
"cluster_name": "elasticsearch",
"version": {
"number": "2.2.0",
"build_hash": "8ff36d139e16f8720f2947ef62c8167a888992fe",
"build_timestamp": "2016-01-27T13:32:39Z",
"build_snapshot": false,
"lucene_version": "5.4.1"
},
"tagline": "You Know, for Search"
}

7. Simple REST API - GET
GET /strutt/property/14523 {
"_index": "strutt",
"_type": "property",
"_id": "14523",
"_version": 1,
"found": true,
"_source": {
"name": "Newton Of Drumduan",
"public_date": "2016-03-20 12:31:31",
"property_address": "Newton Of Drumduan, Dess",
"property_town": "Aboyne",
"property_county": "Aberdeenshire",
"property_postcode": "AB34 5BD",
"property_price_min": 975000,
"property_price_display": "£ 975,000",
"property_features": [
"Garden",
"Rural",
"Outbuildings",
"Land/Paddock"
],
...
...
}
}

8. Simple REST API - POST
POST /strutt/property
{
"name" : "Mark's House",
"property_town" : "Lightwater",
"property_features" : [
"XBox One",
"Big TV",
"Big amp"
]
}
{
"_index": "strutt",
"_type": "property",
"_id": "AVQG5ihvhgvIo9hLVnu2",
"_version": 1,
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": true
}

9. Simple REST API - PUT
PUT /strutt/property/1234
{
"name" : "Mark's House",
"property_town" : "Lightwater",
"property_features" : [
"XBox One",
"Big TV",
"Big amp"
]
}
{
"_index": "strutt",
"_type": "property",
"_id": "1234",
"_version": 1,
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"created": true
}

10. Clever Things Elasticsearch Can Do
Full text search
Geolocation
Aggregations
Custom scoring
Combining elements

11. Full Text Search
GET /strutt/property/_search
{
"query" : {
"match": {
"property_long_description":
"underfloor heating"
}
}
}
{
"took": 5,
"hits": {
"total": 413,
"max_score": 0.94447553,
"hits": [
{
"_id": "15825",
"_score": 0.94447553,
"_source": {
"name": "Lower Sandhurst Road (Lot 1)",
"property_long_description": "Underfloor heating throughout
and Ambiente thermostats..."
}
},
{
"_id": "16783",
"_score": 0.92866411,
"_source": {
"name": "Eaton Place, London",
"property_long_description": "... including underfloor
heating, crestron lighting ..."
}
]
}
}

12. Geolocation - Filter By Distance
GET /strutt/office/_search
{
"query": {
"filtered": {
"filter": {
"bool": {
"filter": [
{
"geo_distance": {
"distance": "10miles",
"distance_type": "sloppy_arc",
"location": {
"lat": "51.41082",
"lon": "-0.67480"
}
}
}
]
}
}
}
}
}

13. Geolocation - Sort By Distance
GET /strutt/city/_search
{
"sort": [
{
"_geo_distance": {
"location": {
"lat": "45.850677",
"lon": "12.389420"
},
"order": "asc",
"unit": "miles",
"distance_type": "sloppy_arc"
}
}
],
"size": 25
}

14. Geolocation - Shape Search
GET /strutt/property/_search
{
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"geo_polygon": {
"location": {
"points": [
{
"lat": 51.381423948522,
"lon": -0.67402839660645
},
{
"lat": 51.379388252364,
"lon": -0.65703392028809
},
{
"lat": 51.370494412739,
"lon": -0.66338539123535
},
...
...

15. Aggregations
Extract data from Elastic results
Three types of aggregation
Metric
Bucket
Pipeline
Why?
Hugely powerful and FAST
GET /strutt/properties/_search
{
"aggregations": {
"average_price" : {
"avg": {
"field" : "price"
}
}
}
}

16. Aggregations - Some Examples
Metric Aggregations
Avg
Geo Bounds
Max/Min/Sum
Bucket Aggregations
Terms
Geo Distance
Range

17. GET /strutt/properties/_search
{
"aggregations": {
"grouped_by_town" : {
"terms" : { "field" : "town" },
"aggregations": {
"average_price" : {
"avg" : { "field" : "price" }
}
}
}
}
}
RESULTS
{
"grouped_by_town": {
"buckets": [{
"key": "Ascot",
"doc_count": 50,
"average_price": { "value": 75 }
}, {
"key": "Windsor",
"doc_count": 20,
"average_price": { "value": 145 }
}]
}
}
Nested Aggregations ?!?!

18. Custom Scoring
GET /strutt/article/_search
{
"query": {
"function_score": {
"query": {
"filtered": {
"filter": { "bool": { "must": [ { "term": { "parent_id": "3030" } } ] } }
}
},
"functions": [
{
"gauss": {
"public_date": {
"origin": "2016-04-12 09:00:07",
"scale": "150d",
"offset": "30d",
"decay": "0.1"
}
}
}
]
}
}
}

19. Combining Elements
GET /strutt/property/_search
{
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"term": {
"property_available": 1
}
},
{
"range": {
"property_price_max": {
"gte": "1000000"
}
}
},
{
"range": {
"property_price_min": {
"lte": "5000000"
}
}
},

20. Combining Elements
{
"geo_polygon": {
"location": {
"points": [
{
"lat": 51.39963404311,
"lon": -0.68990707397461
},
{
"lat": 51.393636224206,
"lon": -0.64115524291992
},
{
"lat": 51.368351060511,
"lon": -0.63600540161133
},
{
"lat": 51.371351725538,
"lon": -0.68750381469727
}
]
}
}
},

21. Combining Elements
{
"range": {
"property_bedrooms": {
"gte": "5"
}
}
},
{
"bool": {
"should": [
{
"term": {
"property_class": "Residential sales"
}
}
]
}
},

22. Combining Elements
{
"bool": {
"should": [
{
"term": {
"property_features": "Swimming Pool"
}
}
]
}
}
],
"filter": [
{
"geo_distance": {
"distance": "10miles",
"distance_type": "sloppy_arc",
"location": {
"lat": "51.41082",
"lon": "-0.67480"
}
}
}
]
}

23. Combining Elements
"sort": [
{
"_geo_distance": {
"location": {
"lat": "51.41082",
"lon": "-0.67480"
},
"order": "asc",
"unit": "miles",
"distance_type": "sloppy_arc"
}
},
{
"property_price_min": {
"order": "desc"
}
}
],
"from": 0,
"size": 10000
}

24. Did I Mention It’s Fast?
{
"took": 12,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"failed": 0
},
"hits": {
"total": 3,
"max_score": null,
"hits": [
...
...

25. And There’s More...
Automatic clustering
Explaining query results
Suggesters
Spelling corrections and fuzzy matching
Match highlighting
Term boosting
Query profiling
Cluster management and monitoring

26. Learn more about Rawnet