![include_recipe "packages"
include_recipe "ruby"
include_recipe "apache2"
if platform?("centos","redhat")
if dist_only?
# just the gem, we'll install the apache module within apache2
package "rubygem-passenger"
return
else
package "httpd-devel"
end
else
%w{ apache2-prefork-dev libapr1-dev }.each do |pkg|
package pkg do
action :upgrade
end
end
end
gem_package "passenger" do
version node[:passenger][:version]
end
execute "passenger_module" do
command 'echo -en "nnnn" | passenger-install-apache2-module'
creates node[:passenger][:module_path]
end](https://image.slidesharecdn.com/systemsbioinformatics-110523225655-phpapp02/75/Systems-Bioinformatics-Workshop-Keynote-53-2048.jpg)
![import boto
import boto.emr
from boto.emr.step import StreamingStep
Connect to Elastic MapReduce
from boto.emr.bootstrap_action import BootstrapAction
import time
# set your aws keys and S3 bucket, e.g. from environment or .boto
AWSKEY=
SECRETKEY=
S3_BUCKET=
NUM_INSTANCES = 1
conn = boto.connect_emr(AWSKEY,SECRETKEY)
bootstrap_step = BootstrapAction("download.tst",
"s3://elasticmapreduce/bootstrap-actions/download.sh",None)
Install packages
step = StreamingStep(name='Wordcount',
mapper='s3n://elasticmapreduce/samples/wordcount/wordSplitter.py',
cache_files = ["s3n://" + S3_BUCKET + "/boto.mod#boto.mod"],
reducer='aggregate',
input='s3n://elasticmapreduce/samples/wordcount/input',
output='s3n://' + S3_BUCKET + '/output/wordcount_output')
Set up mappers &
jobid = conn.run_jobflow(
name="testbootstrap",
reduces
log_uri="s3://" + S3_BUCKET + "/logs",
steps = [step],
bootstrap_actions=[bootstrap_step],
num_instances=NUM_INSTANCES)
print "finished spawning job (note: starting still takes time)"
state = conn.describe_jobflow(jobid).state
print "job state = ", state
print "job id = ", jobid
while state != u'COMPLETED':
print time.localtime() job state
time.sleep(30)
state = conn.describe_jobflow(jobid).state
print "job state = ", state
print "job id = ", jobid
print "final output can be found in s3://" + S3_BUCKET + "/output" + TIMESTAMP
print "try: $ s3cmd sync s3://" + S3_BUCKET + "/output" + TIMESTAMP + " ."](https://image.slidesharecdn.com/systemsbioinformatics-110523225655-phpapp02/75/Systems-Bioinformatics-Workshop-Keynote-54-2048.jpg)
Uploaded byDeepak Singh
Systems Bioinformatics Workshop Keynote
This document discusses how data science platforms can be built on cloud computing infrastructure like Amazon Web Services (AWS). It highlights how AWS provides scalable, on-demand computing and storage resources that allow data and compute needs to scale rapidly. Example applications and customer case studies are presented to show how various organizations are using AWS for large-scale data analysis, including genomics, computational fluid dynamics, and more. The document argues that distributed, programmable cloud infrastructure can support new types of data-driven science by providing massive, rapidly scaling resources.
More Related Content
![[Hatsune Miku] Shoot Frieza with Amazon Kinesis ! [EN]](https://cdn.slidesharecdn.com/ss_thumbnails/jawsdayslt20140315up3en-140321051150-phpapp02-thumbnail.jpg?width=640&height=640&fit=bounds)
![Reproducible datascience [with Terraform]](https://cdn.slidesharecdn.com/ss_thumbnails/reproduceabledatascience1-180630074643-thumbnail.jpg?width=640&height=640&fit=bounds)
![[263] s2graph large-scale-graph-database-with-hbase-2](https://cdn.slidesharecdn.com/ss_thumbnails/236s2graph-large-scale-graph-database-with-hbase-2-150915055019-lva1-app6892-thumbnail.jpg?width=640&height=640&fit=bounds)
![[JSDC 2016] Codex: Conditional Modules Strike Back](https://cdn.slidesharecdn.com/ss_thumbnails/codex-conditionalmodulesstrikebackalexliujsdc2016-161023023432-thumbnail.jpg?width=640&height=640&fit=bounds)
Similar to Systems Bioinformatics Workshop Keynote
Systems Bioinformatics Workshop Keynote
- 1. There is nomagic There is only awesome A platform for data science D e e p a k S i n g h
- 2.
- 3.
- 4.
- 5.
- 6.
- 7. what’s the bigdeal?
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15. to make dataeffective
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 24.
- 25.
- 27. change how wethink about compute
- 28. change how wethink about data
- 29. change how wethink about science
- 30.
- 31.
- 32. dataspaces Further reading: JeffHammerbacher, Information Platforms and the rise of the data scientist, Beautiful Data
- 33. the unreasonable effectiveness of data Halevy, et al. IEEE Intelligent Systems, 24, 8-12 (2009)
- 34. accept all data formats
- 35.
- 36. beyond the database and the data warehouse
- 37. data as a programmable resource
- 38. data as aroyal garden
- 39. compute as afungible commodity
- 40. which brings usto ...
- 41.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49. 3000 CPU’s forone firm’s risk management application 3444JJ' !"#$%&'()'*+,'-./01.2%/' 344'+567/'(.' 8%%9%.:/' 344'JJ' I%:.%/:1=' ;<"&/:1=' A&B:1=' C10"&:1=' C".:1=' E(.:1=' ;"%/:1=' >?,,?,44@' >?,3?,44@' >?,>?,44@' >?,H?,44@' >?,D?,44@' >?,F?,44@' >?,G?,44@'
- 50.
- 51. “infrastructure as code”
- 53. include_recipe "packages" include_recipe "ruby" include_recipe"apache2" if platform?("centos","redhat") if dist_only? # just the gem, we'll install the apache module within apache2 package "rubygem-passenger" return else package "httpd-devel" end else %w{ apache2-prefork-dev libapr1-dev }.each do |pkg| package pkg do action :upgrade end end end gem_package "passenger" do version node[:passenger][:version] end execute "passenger_module" do command 'echo -en "nnnn" | passenger-install-apache2-module' creates node[:passenger][:module_path] end
- 54. import boto import boto.emr fromboto.emr.step import StreamingStep Connect to Elastic MapReduce from boto.emr.bootstrap_action import BootstrapAction import time # set your aws keys and S3 bucket, e.g. from environment or .boto AWSKEY= SECRETKEY= S3_BUCKET= NUM_INSTANCES = 1 conn = boto.connect_emr(AWSKEY,SECRETKEY) bootstrap_step = BootstrapAction("download.tst", "s3://elasticmapreduce/bootstrap-actions/download.sh",None) Install packages step = StreamingStep(name='Wordcount', mapper='s3n://elasticmapreduce/samples/wordcount/wordSplitter.py', cache_files = ["s3n://" + S3_BUCKET + "/boto.mod#boto.mod"], reducer='aggregate', input='s3n://elasticmapreduce/samples/wordcount/input', output='s3n://' + S3_BUCKET + '/output/wordcount_output') Set up mappers & jobid = conn.run_jobflow( name="testbootstrap", reduces log_uri="s3://" + S3_BUCKET + "/logs", steps = [step], bootstrap_actions=[bootstrap_step], num_instances=NUM_INSTANCES) print "finished spawning job (note: starting still takes time)" state = conn.describe_jobflow(jobid).state print "job state = ", state print "job id = ", jobid while state != u'COMPLETED': print time.localtime() job state time.sleep(30) state = conn.describe_jobflow(jobid).state print "job state = ", state print "job id = ", jobid print "final output can be found in s3://" + S3_BUCKET + "/output" + TIMESTAMP print "try: $ s3cmd sync s3://" + S3_BUCKET + "/output" + TIMESTAMP + " ."
- 56. “I terminate the instance and relaunch it. Thats my error handling” Source: @jtimberman on Twitter
- 57. compute is afungible commodity
- 58.
- 59. you can geta lot of awesome
- 60.
- 61.
- 62. S3 Simple Storage Service
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70. s pe ty ce an standard “m1” st in 2 EC high cpu “c1” high memory “m2” http://aws.amazon.com/ec2/instance-types/
- 71.
- 72.
- 73. s pe ty ce an st in cluster compute “cc1” 2 EC cluster GPU “cg1” http://aws.amazon.com/ec2/instance-types/
- 74.
- 75.
- 76.
- 77. Linpack benchmark 880-instance CC1 cluster Performance: 41.82 TFlops* *#231 in Nov 2010 Top 500 rankings
- 78. Credit: K. Jorissen,F. D.Villa, and J. J. Rehr WIEN2k Parallel Performance (U. Washington) KS for huge system at 1 k-point H size 56,000 (25GB) Runtime (16x8 processors) Local (Infiniband) 3h:48 Cloud (10Gbps) 1h:30 ($40) VERY DEMANDING network performance •1200 atom unit cell; SCALAPACK+MPI diagonalization, matrix size 50k-100k
- 79. cost and usemodels
- 80. Ideal Effective Utilization Spot Utilization % Utilization On Demand Utilization Reserved Utilization time
- 81.
- 84.
- 86.
- 87. Ideal Effective Utilization Spot Utilization % Utilization On Demand Utilization Reserved Utilization time
- 88.
- 89.
- 90. everything fails allthe time
- 91.
- 92. new data/compute paradigms
- 94.
- 96.
- 100. Customer Case Study:cyclopic energy OpenFOAM® http://aws.amazon.com/solutions/case-studies/cyclopic-energy/
- 101.
- 102. Credit: Angel Pizzaro,U. Penn
- 103.
- 104. Bioproximity http://aws.amazon.com/solutions/case-studies/bioproximity/
- 106.
- 107. mapreduce for genomics http://bowtie-bio.sourceforge.net/crossbow/index.shtml http://contrail-bio.sourceforge.net http://bowtie-bio.sourceforge.net/myrna/index.shtml
- 108.
- 109.
- 110. large scale data requiresa rethink
- 111.
- 112.
- 113.
- 114.
- 115.
- 116.
- 117. architecture evolved with the internet
- 118. can we builddata science platforms?
- 119. there is nomagic there is only awesome
- 120.
- 121.
- 122.
- 123.
- 124. deesingh@amazon.com Twitter:@mndoci http://slideshare.net/mndoci http://mndoci.com Inspiration and ideas from Matt Wood& Larry Lessig Credit” Oberazzi under a CC-BY-NC-SA license