1fe264456c1af77a24125cc32845b384e37825ad
[sqoop.git] / src / test / org / apache / sqoop / manager / postgresql / PGBulkloadManagerManualTest.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.manager.postgresql;
20
21 import java.io.IOException;
22 import java.sql.Connection;
23 import java.sql.PreparedStatement;
24 import java.sql.SQLException;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.mapred.JobConf;
31 import org.apache.sqoop.testcategories.sqooptest.ManualTest;
32 import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
33 import org.junit.Test;
34
35 import org.apache.sqoop.TestExport;
36 import org.apache.sqoop.mapreduce.db.DBConfiguration;
37 import org.junit.experimental.categories.Category;
38
39
40 /**
41 * Test the PGBulkloadManager implementations.
42 * PGBulkloadManager uses both JDBC driver and pg_bulkload to facilitate it.
43 *
44 * Since this requires a Postgresql installation on your local machine to use,
45 * this class is named in such a way that Hadoop's default QA process does not
46 * run it.
47 *
48 * You need to run this manually with -Dtestcase=PGBulkloadManagerManualTest.
49 *
50 * You need to put Postgresql's JDBC driver library into lib dir.
51 *
52 * You need to create a sqooptest superuser and database and tablespace,
53 * and install pg_bulkload for sqooptest database:
54 *
55 * $ sudo -u postgres createuser -U postgres -s sqooptest
56 * $ sudo -u postgres createdb -U sqooptest sqooptest
57 * $ sudo -u postgres mkdir /var/pgdata/stagingtablespace
58 * $ psql -U sqooptest
59 * -f /usr/local/share/postgresql/contrib/pg_bulkload.sql sqooptest
60 * $ psql -U sqooptest sqooptest
61 * sqooptest=# CREATE USER sqooptest;
62 * sqooptest=# CREATE DATABASE sqooptest;
63 * sqooptest=# CREATE TABLESPACE sqooptest
64 * LOCATION '/var/pgdata/stagingtablespace';
65 * sqooptest=# \q
66 *
67 */
68 @Category({ManualTest.class, PostgresqlTest.class})
69 public class PGBulkloadManagerManualTest extends TestExport {
70
71 public static final Log LOG =
72 LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
73 private DBConfiguration dbConf;
74 static final String HOST_URL =
75 System.getProperty("sqoop.test.postgresql.connectstring.host_url",
76 "jdbc:postgresql://localhost/");
77 static final String DATABASE =
78 System.getProperty("sqoop.test.postgresql.database", "sqooptest");
79 static final String TABLESPACE =
80 System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
81 static final String USERNAME =
82 System.getProperty("sqoop.test.postgresql.username", "sqooptest");
83 static final String PG_BULKLOAD =
84 System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
85 static final String CONNECT_STRING = HOST_URL + DATABASE;
86
87 public PGBulkloadManagerManualTest() {
88 JobConf conf = new JobConf(getConf());
89 DBConfiguration.configureDB(conf,
90 "org.postgresql.Driver",
91 getConnectString(),
92 getUserName(),
93 (String) null, (Integer) null);
94 dbConf = new DBConfiguration(conf);
95 }
96
97
98 @Override
99 protected boolean useHsqldbTestServer() {
100 return false;
101 }
102
103
104 @Override
105 protected String getConnectString() {
106 return CONNECT_STRING;
107 }
108
109
110 protected String getUserName() {
111 return USERNAME;
112 }
113
114
115 @Override
116 protected String getTablePrefix() {
117 return super.getTablePrefix().toLowerCase();
118 }
119
120
121 @Override
122 protected String getTableName() {
123 return super.getTableName().toLowerCase();
124 }
125
126 @Override
127 public String getStagingTableName() {
128 return super.getStagingTableName().toLowerCase();
129 }
130
131
132 @Override
133 protected Connection getConnection() {
134 try {
135 Connection conn = dbConf.getConnection();
136 conn.setAutoCommit(false);
137 PreparedStatement stmt =
138 conn.prepareStatement("SET extra_float_digits TO 0");
139 stmt.executeUpdate();
140 conn.commit();
141 return conn;
142 } catch (SQLException sqlE) {
143 LOG.error("Could not get connection to test server: " + sqlE);
144 return null;
145 } catch (ClassNotFoundException cnfE) {
146 LOG.error("Could not find driver class: " + cnfE);
147 return null;
148 }
149 }
150
151
152 @Override
153 protected String getDropTableStatement(String tableName) {
154 return "DROP TABLE IF EXISTS " + tableName;
155 }
156
157
158 @Override
159 protected String[] getArgv(boolean includeHadoopFlags,
160 int rowsPerStatement,
161 int statementsPerTx,
162 String... additionalArgv) {
163 ArrayList<String> args =
164 new ArrayList<String>(Arrays.asList(additionalArgv));
165 args.add("-D");
166 args.add("pgbulkload.bin=" + PG_BULKLOAD);
167 args.add("--username");
168 args.add(getUserName());
169 args.add("--connection-manager");
170 args.add("org.apache.sqoop.manager.PGBulkloadManager");
171 args.add("--staging-table");
172 args.add("dummy");
173 args.add("--clear-staging-table");
174 return super.getArgv(includeHadoopFlags,
175 rowsPerStatement,
176 statementsPerTx,
177 args.toArray(new String[0]));
178 }
179
180
181 @Override
182 protected String [] getCodeGenArgv(String... extraArgs) {
183 ArrayList<String> args = new ArrayList<String>(Arrays.asList(extraArgs));
184 args.add("--username");
185 args.add(getUserName());
186 return super.getCodeGenArgv(args.toArray(new String[0]));
187 }
188
189
190 @Override
191 public void testColumnsExport() throws IOException, SQLException {
192 // PGBulkloadManager does not support --columns option.
193 }
194
195 @Test
196 public void testMultiReduceExport() throws IOException, SQLException {
197 multiFileTest(2, 10, 2, "-D", "mapred.reduce.tasks=2");
198 }
199
200 @Test
201 public void testMultiReduceExportWithNewProp()
202 throws IOException, SQLException {
203 multiFileTest(2, 10, 2, "-D", "mapreduce.job.reduces=2");
204 }
205
206 @Test
207 public void testExportWithTablespace() throws IOException, SQLException {
208 multiFileTest(1, 10, 1,
209 "-D", "pgbulkload.staging.tablespace=" + TABLESPACE);
210 }
211 }