Consuming Nordic Statbank data with JSON-stat

1,198 views

Published on

Master Class at the Nordiskt statistikermöte 2016. Stockholm, August 22, 2016.

Published in: Data & Analytics

Consuming Nordic Statbank data with JSON-stat

  1. 1. Consuming Nordic Statbank data with JSON-stat Xavier Badosa (@badosa) Statistical Institute of Catalonia (Idescat)
  2. 2. Aluu Hallå Hallo Halló Hei Hej
  3. 3. Consuming Nordic Statbank data with JSON-stat Xavier Badosa (@badosa) Statistical Institute of Catalonia (Idescat)
  4. 4. Consuming Nordic Statbank data with JSON-stat
  5. 5. JSON-stat.org a simple light standard for all kinds of data disseminators }
  6. 6. http://www.stat2go.com
  7. 7. data as an infrastructure
  8. 8. API NSO as a platform data as an infrastructure
  9. 9. The Software Staircase access processing webpages
  10. 10. download The Software Staircase access processing webpages
  11. 11. webpages download API The Software Staircase access processing
  12. 12. data provider developer
  13. 13. users data provider developer
  14. 14. users data app
  15. 15. users data API app
  16. 16. real tim e
  17. 17. lightweight
  18. 18. Statistics Sweden AM0401DA Population aged 15-74 (LFS), thousands by sex, age, labour status and month SDMX-ML XML PX-JSON JSON JSON-stat JSON 3,709 Kb 848 Kb 92 Kb
  19. 19. JavaScript Object Notation JSON http://json.org
  20. 20. "This is a text." JS
  21. 21. 666 JS
  22. 22. 666 3.14159265 JS
  23. 23. 666 3.14159265 -299792458 JS
  24. 24. 666 3.14159265 -299792458 null JS
  25. 25. 666 3.14159265 -299792458 null true false JS
  26. 26. [ 1, 2, 3 ] JS
  27. 27. [ 1, 2, 3 ] [ "one", "two", "three" ]a = JS
  28. 28. [ 1, 2, 3 ] [ "one", "two", "three" ]a = a[1] 0 1 2 JS
  29. 29. [ "one", "two", "three" ] [ 1, 2, 3 ] [ "Men", 301784 ] JS
  30. 30. [ "one", "two", "three" ] [ 1, 2, 3 ] [ "Men", 301784 ] [ [ "Men", 301784 ], [ "Women", 284434 ] ] JS
  31. 31. [ [ "Men", "0-4", "2015", "Sweden", 301784 ], [ "Women", "0-4", "2015", "Sweden", 284434 ] ] JS
  32. 32. [ { "sex": "Men", "val": 301784 } {"sex": "Women", "value": 3800} ] a = JS
  33. 33. [ { "sex": "Men", "val": 301784 } {"sex": "Women", "value": 3800} ] a = a["val"] a.val JS
  34. 34. [ { "sex": "Men", "val": 301784 }, { "sex": "Women", "val": 284434 } ] JS
  35. 35. [ { "sex": "Men", "val": 301784 }, { "sex": "Women", "val": 284434 } ] a = JS
  36. 36. [ { "sex": "Men", "val": 301784 }, { "sex": "Women", "val": 284434 } ] a = a[1] JS
  37. 37. [ { "sex": "Men", "val": 301784 }, { "sex": "Women", "val": 284434 } ] a = a[1]["sex"] a[1].sex JS
  38. 38. [ { "concept": "Population", "sex": "Men", "age": "0-4", "country": "Sweden", "year": "2015", "value": 301784 }, { "concept": "Population", "sex": "Women", "age": "0-4", "country": "Sweden", "year": "2015", "value": 284434 }, ... ] Array-of-objects pattern
  39. 39. Abbreviation
  40. 40. Cubic Model Describe data in dimension terms
  41. 41. [ { "concept": "Population", "sex": "Men", "age": "0-4", "country": "Sweden", "year": "2015", "value": 301784 }, { "concept": "Population", "sex": "Women", "age": "0-4", "country": "Sweden", "year": "2015", "value": 284434 }, ... ]
  42. 42. [ ] , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , "value" :
  43. 43. value note/source/updated label
  44. 44. [ ] , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , "value" : "version" : "2.0", "class" : "dataset", "label" : "Population by sex and age group. Canada. 2012", "source" : "Statistics Canada, CANSIM, table 051-0001", "updated" : "2012-09-27", } {
  45. 45. [ ] , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , "value" : } "id" : [ "country" , "year" , "age" , "concept" , "sex" ], "size" : [ 1 , 1 , 20 , 2 , 3 ], "dimension" : { … } "version" : "2.0", "class" : "dataset", "label" : "Population by sex and age group. Canada. 2012", "source" : "Statistics Canada, CANSIM, table 051-0001", "updated" : "2012-09-27", {
  46. 46. "value" : [ … ], } "id" : [ "country" , "year" , "age" , "concept" , "sex" ], "size" : [ 1 , 1 , 20 , 2 , 3 ], "role" : { "time" : ["year"] , "geo" : ["country"] , "metric" : ["concept"] }, "dimension" : { "country" : { … }, "year" : { … }, "age" : { … }, "concept" : { … }, "sex" : { … } } "version" : "2.0", "class" : "dataset", "label" : "Population by sex and age group. Canada. 2012", "source" : "Statistics Canada, CANSIM, table 051-0001", "updated" : "2012-09-27", {
  47. 47. "value" : [ … ], } "id" : [ "country" , "year" , "age" , "concept" , "sex" ], "size" : [ 1 , 1 , 20 , 2 , 3 ], "role" : { "time" : ["year"] , "geo" : ["country"] , "metric" : ["concept"] }, "dimension" : { "country" : { … }, "year" : { … }, "age" : { … }, "concept" : { … }, "sex" : { … } } "version" : "2.0", "class" : "dataset", "label" : "Population by sex and age group. Canada. 2012", "source" : "Statistics Canada, CANSIM, table 051-0001", "updated" : "2012-09-27", {
  48. 48. "sex" : { "label" : "sex", "category" : { "index" : ["T", "M", "F"], "label" : { "T" : "total", "M" : "male", "F" : "female" } } }
  49. 49. [ ] , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ? The “unflattening” problem
  50. 50. function arr2num( arr, size ){ for(var i=0, num=0, mult=1, ndims=size.length; i<ndims; i++){ mult*=(i>0) ? size[ndims-i] : 1; num+=mult*arr[ndims-i-1]; } return num; } The “unflattening” problem
  51. 51. Method: Row-major order In computing, row-major order and column- major order describe methods for arranging multidimensional arrays in linear storage such as memory. The “unflattening” problem
  52. 52. users data API dev
  53. 53. users dev data tools API
  54. 54. JavaScript R Python Python Java
  55. 55. You could not step twice into the same river. Everything changes and nothing stands still. Heraclitus
  56. 56. Responses Version bundle 0 (set of one or more datasets)
  57. 57. Responses Version bundle 1dataset dimension collection
  58. 58. Responses Version bundle 2dataset dimension collection
  59. 59. 2.0 2.0 2.0 2.0 JavaScript R Python Python Java
  60. 60. { "version": "2.0", "class": "dataset", "label": "Persons by region, sex, age and time", "source": "Statistics Norway", "updated": "2016-06-01T19:45:12Z", "value": [ 26175, 26923, 27394, 29332, 30325, ... ], "id": ["Region","Kjonn","Alder","Tid","ContentsCode"], "size": [1, 2, 106, 31, 1], "role": {"time": ["Tid"], "metric": ["ContentsCode"]}, "dimension": {...} } Version 2. Dataset
  61. 61. { "your dataset id here": { "label": "Persons by region, sex, age and time", "source": "Statistics Norway", "updated": "2016-06-01T19:45:12Z", "value": [ 26175, 26923, 27394, 29332, 30325, ... ], "dimension": {...} } } Version 0/1. Bundle
  62. 62. { "dataset": { "label": "Persons by region, sex, age and time", "source": "Statistics Norway", "updated": "2016-06-01T19:45:12Z", "value": [ 26175, 26923, 27394, 29332, 30325, ... ], "dimension": {...} } } Version 0/1. Bundle
  63. 63. PX-Web
  64. 64. PX-Web
  65. 65. JSON-stat
  66. 66. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA
  67. 67. user dev prov
  68. 68. interface data logic
  69. 69. interface logic retrieve process build data
  70. 70. process build retrieve JavaScript
  71. 71. process build retrieve JavaScript
  72. 72. process build retrieve JSON-stat Javascript Toolkit JSON-stat Javascript Utilities Suite https://json-stat.com JavaScript
  73. 73. process build JavaScript JSONP, XHR, Fetch JJT jQuery… JJT retrieve
  74. 74. process build JavaScript JSONP, XHR, Fetch JJT jQuery… JJT Visual retrieve
  75. 75. Introducing the JSON-stat Javascript Toolkit json-stat.com
  76. 76. JSON-stat Tree Dataset Dimension Category Data 1. Chainable traversing methods
  77. 77. Dataset Dimension Category Data Metadata Path class label length id 1. Chainable traversing methods
  78. 78. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Dimension(3).Category(2).label JSONstat(obj).Dataset("dataset").Dimension("sex").Category("F").label "female"
  79. 79. Dataset Dimension Category Data Data Path value status 1. Chainable traversing methods
  80. 80. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { } ).value
  81. 81. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { "country" : "CA", "year" : "2012", "concept" : "POP", "age" : "34", "sex" : "F" } ).value 1202.8
  82. 82. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { "country" : "CA", "year" : "2012", "concept" : "POP", "age" : "34", "sex" : "F" } ).value 1202.8
  83. 83. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { "country" : "CA", "year" : "2012", "concept" : "POP", "age" : "34", "sex" : "F" } )
  84. 84. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { "country" : "CA", "year" : "2012", "concept" : "POP", "age" : "34", "sex" : "F" }, false ) [2406.3, 1203.5, 1202.8] T M F
  85. 85. 1. Chainable traversing methods JSONstat(obj).Dataset(0).Data( { country : "CA", year : "2012", concept : "POP", age : "34", sex : "F" }, false ) [2406.3, 1203.5, 1202.8] T M F
  86. 86. 2. Chainable transformation methods Dataset toTable Slice
  87. 87. JSONstat( obj ).Dataset(0).toTable( { type : "arrobj" } ); [ { age : "Total", concept : "Persons (thousands)", country : "Canada", sex : "Total", year : "2012", value : 34880.5 }, { age : "Total", concept : "Persons (thousands)", country : "Canada", sex : "Male", year : "2012", value : 17309.1 }, … ] Array-of-objects pattern
  88. 88. json-stat.com/nsm
  89. 89. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA
  90. 90. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Download
  91. 91. process build JavaScript JSONP, XHR, Fetch JJT jQuery… JJT Visual retrieve
  92. 92. Demo
  93. 93. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API Dedicated Section
  94. 94. http://www.scb.se/en_/About-us/Open-data-API/API-for-the-Statistical-Database-/
  95. 95. Demo
  96. 96. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API JSONP
  97. 97. Demo
  98. 98. HTTP Verbs GET POST Requests that the server stores the data enclosed in the body of the request message. Requests a representation of the specified resource. …
  99. 99. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GETJSONP
  100. 100. Demo
  101. 101. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GET SQueryJSONP
  102. 102. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GETJSONP SQuery
  103. 103. Demo
  104. 104. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GETJSONP SQuery
  105. 105. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands SBA CORS SQuery
  106. 106. Cross-Origin Resource Sharing A web application using XHR/Fetch can only make HTTP requests to its own domain. Same-Origin Policy CORS SQuery
  107. 107. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GETJSONP SQuery
  108. 108. Demo
  109. 109. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GETJSONP SQuery
  110. 110. Demo
  111. 111. Denmark Norway Sweden Finland Greenland Iceland Faroe Islands ΩΩ SBA Down. API GET SQueryJSONP
  112. 112. ΩΩThank You
  113. 113. Credits “Soma” (blocks’ background) by Dru! (CC BY-NC) “Metal movable type” by Willi Heidelbach (CC BY-SA) “Portrait” (cubic head) by Thomas Leth-Olsen (CC BY) “River in Iceland” by Kamil Porembiński (CC BY-SA) “Silicon 0116 6336” (integrated circuit) by Ross Elliott (CC BY)
 “Sterile” (walking girl) by Lee Nachtigal (CC BY) “Cubes” (sculpture) by Alex [Fino] LA (CC BY-SA) “Railroad” by Xavier Badosa (CC BY) “Dartboard” by Jacob Vance (CC BY-NC) Icons by Visualpharm.com (Linkware)

×