Detail instructions about installing Sqoop2 server on Ubuntu ,integrating with Hadoop and java application.

1. Hadoop - Sqoop2 Server Setup And Application Integration Page 1
Hadoop - Sqoop2 Server Setup And Application Integration
Introduction...................................................................................................................................1
Server Setup...................................................................................................................................1
Java Client Setup.............................................................................................................................2
Sqoop Client Code:......................................................................................................................2
Eclipse Reference Jars..................................................................................................................4
Introduction
 Sqoop2serversetuponHadoop 2.2.1.
 Sqoop2java clientexample.
 Supportingjarfiles.
Server Setup
1) Downloadsqoop-1.99.4-bin-hadoop200.tar.gzandextract. [Compatible toHadoop2.2]
2) Do the belowmodificationsinsqoop.propertiesandcatalina.propertiesin/server/conf folder.
3) Update catalina.propertieswithhadoopinstallationlibrarieslocation(E.g:Myhadoopinstall
locationis/usr/local/hadoop]
Catalina.properties:
4) Update sqoop.propertieswithhadoopconf[e.g:/usr/local/hadoop/etc/conf] folderlocation.
Sqoop.properties:

2. Hadoop - Sqoop2 Server Setup And Application Integration Page 2
5) Execute /server/bin/./sqoop2-toolverifyanditshouldreturn "Success"message.
6) Start sqoopserver. /server/bin/./sqoop2-severstart.
Note:
 Sqoop1 isclientonlymodel andsqoop2isclientserver model withenhancedsupports.
 Sqoop2bundledwithtomcatserver6.0 to supportREST services.
 Sqoop2importOLTP data in to HDFS filesandthere isnoconnectortechnologiestodirectly
importdata fromOLTP to HBASE. (Developmentisinprogress)
Java Client Setup
1) Download sqoop-client-1.99.4.jarandjersey-bundle-1.18.3.jarfrommavenrepository.
2) Supportinglibraries ineclipse (Maynotrequire all) referbelow section.
3) Sample OLTPdatabase table importtoHadoop file.
SqoopClientCode:
import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
import org.apache.sqoop.validation.Status;
public classHBeatToHDFS {
public static void main(String[]args) {
String connectionString = "jdbc:oracle:thin: database:1528/DEMO";

3. Hadoop - Sqoop2 Server Setup And Application Integration Page 3
String username = "YourOLTPUserName";
String password= "YourOLTPPassword";
String schemaName = "YourOLTPDB";
String tableName = "EDI_TICKETS";
String partitionColumn = "CONTROL_NUMBER";
String outputDirectory= "/usr/local";
String url = "http://YourSqoopIP:12000/sqoop/";
String hdfsURI = "hdfs://localhost:9000/";
SqoopClient client = newSqoopClient(url);
long fromConnectorId= 2;
MLink fromLink= client.createLink(fromConnectorId);
fromLink.setName("JDBCconnector1");
fromLink.setCreationUser("test");
MLinkConfigfromLinkConfig = fromLink.getConnectorLinkConfig();
fromLinkConfig.getStringInput("linkConfig.connectionString").setValue(connectionString);
fromLinkConfig.getStringInput("linkConfig.jdbcDriver").setValue("oracle.jdbc.OracleDriver");
fromLinkConfig.getStringInput("linkConfig.username").setValue(username);
fromLinkConfig.getStringInput("linkConfig.password").setValue(password);
Status fromStatus = client.saveLink(fromLink);
if (fromStatus.canProceed()){
System.out.println("JDBCLink,ID:" + fromLink.getPersistenceId());
} else {
System.out.println("JDBCLink");
}
// create HDFS connector
long toConnectorId= 1;
MLink toLink = client.createLink(toConnectorId);
toLink.setName("HDFSconnector");
toLink.setCreationUser("test");
MLinkConfigtoLinkConfig = toLink.getConnectorLinkConfig();
toLinkConfig.getStringInput("linkConfig.uri").setValue(hdfsURI);
Status toStatus = client.saveLink(toLink);
if (toStatus.canProceed()) {
System.out.println("HDFSLink,ID:" + toLink.getPersistenceId());
} else {
System.out.println("HDFSLink");
}
long fromLinkId= fromLink.getPersistenceId();
long toLinkId= toLink.getPersistenceId();
MJob job = client.createJob(fromLinkId, toLinkId);//create job withjdbc and hdfs links
job.setName("MySQL to HDFSjob");
job.setCreationUser("test");
MFromConfigfromJobConfig= job.getFromJobConfig();
fromJobConfig.getStringInput("fromJobConfig.schemaName").setValue(schemaName);
fromJobConfig.getStringInput("fromJobConfig.tableName").setValue(tableName);
fromJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue(partitionColumn);
MToConfig toJobConfig = job.getToJobConfig();
toJobConfig.getStringInput("toJobConfig.outputDirectory").setValue(outputDirectory);
Status status = client.saveJob(job);
if (status.canProceed()) {
System.out.println("JOB,ID:" + job.getPersistenceId());
} else {
System.out.println("Job can't be created");
}
long jobId= job.getPersistenceId();
MSubmissionsubmission= client.startJob(jobId);
System.out.println("JOB :" + submission.getStatus());
while (submission.getStatus().isRunning()

4. Hadoop - Sqoop2 Server Setup And Application Integration Page 4
&& submission.getProgress()!= -1) {
System.out.println("JOB:"
+ String.format("%.2f %%", submission.getProgress() * 100));
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("JOB... ...");
System.out.println("Hadoop:" + submission.getExternalId());
Counters counters = submission.getCounters();
if (counters != null){
System.out.println(":");
for (CounterGroupgroup:counters) {
System.out.print("t");
System.out.println(group.getName());
for (Counter counter :group) {
System.out.print("tt");
System.out.print(counter.getName());
System.out.print(":");
System.out.println(counter.getValue());
}
}
}
if (submission.getExceptionInfo() != null) {
System.out.println("JOB:" + submission.getExceptionInfo());
}
System.out.println("sqoopjob successfullysubmitted");
}
}
EclipseReferenceJars
/usr/local/hadoop/share/hadoop/yarn/lib/aopalliance-1.0.jar
/usr/local/hadoop/share/hadoop/yarn/lib/asm-3.2.jar
/usr/local/hadoop/share/hadoop/yarn/lib/avro-1.7.4.jar
/usr/local/hadoop/share/hadoop/yarn/lib/commons-compress-1.4.1.jar
/usr/local/hadoop/share/hadoop/yarn/lib/commons-io-2.1.jar
/usr/local/hadoop/share/hadoop/yarn/lib/guice-3.0.jar
/usr/local/hadoop/share/hadoop/yarn/lib/guice-servlet-3.0.jar
/usr/local/hadoop/share/hadoop/yarn/lib/hadoop-annotations-2.2.0.jar
/usr/local/hadoop/share/hadoop/yarn/lib/hamcrest-core-1.1.jar
/usr/local/hadoop/share/hadoop/yarn/lib/jackson-core-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/yarn/lib/jackson-mapper-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/yarn/lib/javax.inject-1.jar
/usr/local/hadoop/share/hadoop/yarn/lib/jersey-core-1.9.jar
/usr/local/hadoop/share/hadoop/yarn/lib/jersey-guice-1.9.jar
/usr/local/hadoop/share/hadoop/yarn/lib/jersey-server-1.9.jar
/usr/local/hadoop/share/hadoop/yarn/lib/junit-4.10.jar
/usr/local/hadoop/share/hadoop/yarn/lib/log4j-1.2.17.jar
/usr/local/hadoop/share/hadoop/yarn/lib/netty-3.6.2.Final.jar
/usr/local/hadoop/share/hadoop/yarn/lib/paranamer-2.3.jar
/usr/local/hadoop/share/hadoop/yarn/lib/protobuf-java-2.5.0.jar
/usr/local/hadoop/share/hadoop/yarn/lib/snappy-java-1.0.4.1.jar
/usr/local/hadoop/share/hadoop/yarn/lib/xz-1.0.jar

5. Hadoop - Sqoop2 Server Setup And Application Integration Page 5
/home/rajekra/spark_1.4_JARS/SQOOP_JARS/sqoop-client-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/commons-cli-1.2.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/commons-dbcp-1.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/commons-io-2.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/commons-lang-2.5.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/commons-pool-1.5.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/connector-sdk-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/derby-10.8.2.2.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/guava-11.0.2.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/hamcrest-core-1.3.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/joda-time-2.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/json-simple-1.1.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/jsr305-1.3.9.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/junit-4.11.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/log4j-1.2.16.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-common-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-connector-generic-jdbc-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-connector-hdfs-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-core-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-execution-mapreduce-1.99.4-hadoop2
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-repository-derby-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-spi-1.99.4.jar
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-submission-mapreduce-1.99.4-hadoop
/usr/local/sqoop/server/webapps/sqoop/WEB-INF/lib/sqoop-tools-1.99.4.jar
/usr/local/hadoop/share/hadoop/common/hadoop-common-2.2.0.jar
/usr/local/hadoop/share/hadoop/common/hadoop-common-2.2.0-tests.jar
/usr/local/hadoop/share/hadoop/common/hadoop-nfs-2.2.0.jar
/usr/local/hadoop/share/hadoop/common/lib/activation-1.1.jar
/usr/local/hadoop/share/hadoop/common/lib/asm-3.2.jar
/usr/local/hadoop/share/hadoop/common/lib/avro-1.7.4.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-beanutils-1.7.0.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-beanutils-core-1.8.0.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-cli-1.2.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-codec-1.4.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-collections-3.2.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-compress-1.4.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-configuration-1.6.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-digester-1.8.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-el-1.0.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-httpclient-3.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-io-2.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-lang-2.5.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-logging-1.1.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-math-2.1.jar
/usr/local/hadoop/share/hadoop/common/lib/commons-net-3.1.jar
/usr/local/hadoop/share/hadoop/common/lib/guava-11.0.2.jar
/usr/local/hadoop/share/hadoop/common/lib/hadoop-annotations-2.2.0.jar
/usr/local/hadoop/share/hadoop/common/lib/hadoop-auth-2.2.0.jar
/usr/local/hadoop/share/hadoop/common/lib/jackson-core-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/common/lib/jackson-jaxrs-1.8.8.jar
/usr/local/hadoop/share/hadoop/common/lib/jackson-mapper-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/common/lib/jackson-xc-1.8.8.jar
/usr/local/hadoop/share/hadoop/common/lib/jasper-compiler-5.5.23.jar
/usr/local/hadoop/share/hadoop/common/lib/jasper-runtime-5.5.23.jar
/usr/local/hadoop/share/hadoop/common/lib/jaxb-api-2.2.2.jar
/usr/local/hadoop/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar
/usr/local/hadoop/share/hadoop/common/lib/jersey-core-1.9.jar
/usr/local/hadoop/share/hadoop/common/lib/jersey-json-1.9.jar

6. Hadoop - Sqoop2 Server Setup And Application Integration Page 6
/usr/local/hadoop/share/hadoop/common/lib/jersey-server-1.9.jar
/usr/local/hadoop/share/hadoop/common/lib/jets3t-0.6.1.jar
/usr/local/hadoop/share/hadoop/common/lib/jettison-1.1.jar
/usr/local/hadoop/share/hadoop/common/lib/jetty-6.1.26.jar
/usr/local/hadoop/share/hadoop/common/lib/jetty-util-6.1.26.jar
/usr/local/hadoop/share/hadoop/common/lib/jsch-0.1.42.jar
/usr/local/hadoop/share/hadoop/common/lib/jsp-api-2.1.jar
/usr/local/hadoop/share/hadoop/common/lib/jsr305-1.3.9.jar
/usr/local/hadoop/share/hadoop/common/lib/junit-4.8.2.jar
/usr/local/hadoop/share/hadoop/common/lib/log4j-1.2.17.jar
/usr/local/hadoop/share/hadoop/common/lib/mockito-all-1.8.5.jar
/usr/local/hadoop/share/hadoop/common/lib/netty-3.6.2.Final.jar
/usr/local/hadoop/share/hadoop/common/lib/paranamer-2.3.jar
/usr/local/hadoop/share/hadoop/common/lib/protobuf-java-2.5.0.jar
/usr/local/hadoop/share/hadoop/common/lib/servlet-api-2.5.jar
/usr/local/hadoop/share/hadoop/common/lib/slf4j-api-1.7.5.jar
/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar
/usr/local/hadoop/share/hadoop/common/lib/snappy-java-1.0.4.1.jar
/usr/local/hadoop/share/hadoop/common/lib/stax-api-1.0.1.jar
/usr/local/hadoop/share/hadoop/common/lib/xmlenc-0.52.jar
/usr/local/hadoop/share/hadoop/common/lib/xz-1.0.jar
/usr/local/hadoop/share/hadoop/common/lib/zookeeper-3.4.5.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/aopalliance-1.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/asm-3.2.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/avro-1.7.4.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/commons-compress-1.4.1.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/commons-io-2.1.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/guice-3.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/guice-servlet-3.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/hadoop-annotations-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/hamcrest-core-1.1.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/jackson-core-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/jackson-mapper-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/javax.inject-1.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/jersey-core-1.9.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/jersey-guice-1.9.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/jersey-server-1.9.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/junit-4.10.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/log4j-1.2.17.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/netty-3.6.2.Final.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/paranamer-2.3.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/protobuf-java-2.5.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/snappy-java-1.0.4.1.jar
/usr/local/hadoop/share/hadoop/mapreduce/lib/xz-1.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-app-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-common-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-plugins-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-2.2.0-tests.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-2.2.0.jar
/usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/asm-3.2.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-cli-1.2.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-codec-1.4.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-daemon-1.0.13.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-el-1.0.jar

7. Hadoop - Sqoop2 Server Setup And Application Integration Page 7
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-io-2.1.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-lang-2.5.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/commons-logging-1.1.1.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/guava-11.0.2.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jackson-core-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jackson-mapper-asl-1.8.8.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jasper-runtime-5.5.23.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jersey-core-1.9.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jersey-server-1.9.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jetty-6.1.26.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jetty-util-6.1.26.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jsp-api-2.1.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/jsr305-1.3.9.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/log4j-1.2.17.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/netty-3.6.2.Final.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/protobuf-java-2.5.0.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/servlet-api-2.5.jar
/usr/local/hadoop/share/hadoop/hdfs/lib/xmlenc-0.52.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-archives-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-datajoin-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-distcp-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-extras-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-gridmix-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-rumen-2.2.0.jar
/usr/local/hadoop/share/hadoop/tools/lib/hadoop-streaming-2.2.0.jar
/home/rajekra/spark_1.4_JARS/SQOOP_JARS/jersey-bundle-1.18.3.jar