Published site at d5aaeee88b331e064830a2774f4fed238631457c.
[hbase-site.git] / testdevapidocs / src-html / org / apache / hadoop / hbase / backup / TestIncrementalBackupMergeWithFailures.html
1 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2 <html lang="en">
3 <head>
4 <title>Source code</title>
5 <link rel="stylesheet" type="text/css" href="../../../../../../stylesheet.css" title="Style">
6 </head>
7 <body>
8 <div class="sourceContainer">
9 <pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a>
10 <span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one<a name="line.2"></a>
11 <span class="sourceLineNo">003</span> * or more contributor license agreements. See the NOTICE file<a name="line.3"></a>
12 <span class="sourceLineNo">004</span> * distributed with this work for additional information<a name="line.4"></a>
13 <span class="sourceLineNo">005</span> * regarding copyright ownership. The ASF licenses this file<a name="line.5"></a>
14 <span class="sourceLineNo">006</span> * to you under the Apache License, Version 2.0 (the<a name="line.6"></a>
15 <span class="sourceLineNo">007</span> * "License"); you may not use this file except in compliance<a name="line.7"></a>
16 <span class="sourceLineNo">008</span> * with the License. You may obtain a copy of the License at<a name="line.8"></a>
17 <span class="sourceLineNo">009</span> *<a name="line.9"></a>
18 <span class="sourceLineNo">010</span> * http://www.apache.org/licenses/LICENSE-2.0<a name="line.10"></a>
19 <span class="sourceLineNo">011</span> *<a name="line.11"></a>
20 <span class="sourceLineNo">012</span> * Unless required by applicable law or agreed to in writing, software<a name="line.12"></a>
21 <span class="sourceLineNo">013</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.13"></a>
22 <span class="sourceLineNo">014</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.14"></a>
23 <span class="sourceLineNo">015</span> * See the License for the specific language governing permissions and<a name="line.15"></a>
24 <span class="sourceLineNo">016</span> * limitations under the License.<a name="line.16"></a>
25 <span class="sourceLineNo">017</span> */<a name="line.17"></a>
26 <span class="sourceLineNo">018</span>package org.apache.hadoop.hbase.backup;<a name="line.18"></a>
27 <span class="sourceLineNo">019</span><a name="line.19"></a>
28 <span class="sourceLineNo">020</span>import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded;<a name="line.20"></a>
29 <span class="sourceLineNo">021</span>import static org.junit.Assert.assertFalse;<a name="line.21"></a>
30 <span class="sourceLineNo">022</span>import static org.junit.Assert.assertTrue;<a name="line.22"></a>
31 <span class="sourceLineNo">023</span><a name="line.23"></a>
32 <span class="sourceLineNo">024</span>import java.io.IOException;<a name="line.24"></a>
33 <span class="sourceLineNo">025</span>import java.util.ArrayList;<a name="line.25"></a>
34 <span class="sourceLineNo">026</span>import java.util.List;<a name="line.26"></a>
35 <span class="sourceLineNo">027</span>import org.apache.commons.lang3.StringUtils;<a name="line.27"></a>
36 <span class="sourceLineNo">028</span>import org.apache.hadoop.conf.Configuration;<a name="line.28"></a>
37 <span class="sourceLineNo">029</span>import org.apache.hadoop.fs.FileSystem;<a name="line.29"></a>
38 <span class="sourceLineNo">030</span>import org.apache.hadoop.fs.Path;<a name="line.30"></a>
39 <span class="sourceLineNo">031</span>import org.apache.hadoop.hbase.HBaseClassTestRule;<a name="line.31"></a>
40 <span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.TableName;<a name="line.32"></a>
41 <span class="sourceLineNo">033</span>import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;<a name="line.33"></a>
42 <span class="sourceLineNo">034</span>import org.apache.hadoop.hbase.backup.impl.BackupCommands;<a name="line.34"></a>
43 <span class="sourceLineNo">035</span>import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;<a name="line.35"></a>
44 <span class="sourceLineNo">036</span>import org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupMergeJob;<a name="line.36"></a>
45 <span class="sourceLineNo">037</span>import org.apache.hadoop.hbase.backup.mapreduce.MapReduceHFileSplitterJob;<a name="line.37"></a>
46 <span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.backup.util.BackupUtils;<a name="line.38"></a>
47 <span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.39"></a>
48 <span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.client.ConnectionFactory;<a name="line.40"></a>
49 <span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.client.HBaseAdmin;<a name="line.41"></a>
50 <span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.client.HTable;<a name="line.42"></a>
51 <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.client.Table;<a name="line.43"></a>
52 <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.testclassification.LargeTests;<a name="line.44"></a>
53 <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.45"></a>
54 <span class="sourceLineNo">046</span>import org.junit.Assert;<a name="line.46"></a>
55 <span class="sourceLineNo">047</span>import org.junit.ClassRule;<a name="line.47"></a>
56 <span class="sourceLineNo">048</span>import org.junit.Test;<a name="line.48"></a>
57 <span class="sourceLineNo">049</span>import org.junit.experimental.categories.Category;<a name="line.49"></a>
58 <span class="sourceLineNo">050</span>import org.slf4j.Logger;<a name="line.50"></a>
59 <span class="sourceLineNo">051</span>import org.slf4j.LoggerFactory;<a name="line.51"></a>
60 <span class="sourceLineNo">052</span><a name="line.52"></a>
61 <span class="sourceLineNo">053</span>import org.apache.hbase.thirdparty.com.google.common.collect.Lists;<a name="line.53"></a>
62 <span class="sourceLineNo">054</span><a name="line.54"></a>
63 <span class="sourceLineNo">055</span>@Category(LargeTests.class)<a name="line.55"></a>
64 <span class="sourceLineNo">056</span>public class TestIncrementalBackupMergeWithFailures extends TestBackupBase {<a name="line.56"></a>
65 <span class="sourceLineNo">057</span><a name="line.57"></a>
66 <span class="sourceLineNo">058</span> @ClassRule<a name="line.58"></a>
67 <span class="sourceLineNo">059</span> public static final HBaseClassTestRule CLASS_RULE =<a name="line.59"></a>
68 <span class="sourceLineNo">060</span> HBaseClassTestRule.forClass(TestIncrementalBackupMergeWithFailures.class);<a name="line.60"></a>
69 <span class="sourceLineNo">061</span><a name="line.61"></a>
70 <span class="sourceLineNo">062</span> private static final Logger LOG =<a name="line.62"></a>
71 <span class="sourceLineNo">063</span> LoggerFactory.getLogger(TestIncrementalBackupMergeWithFailures.class);<a name="line.63"></a>
72 <span class="sourceLineNo">064</span><a name="line.64"></a>
73 <span class="sourceLineNo">065</span> enum FailurePhase {<a name="line.65"></a>
74 <span class="sourceLineNo">066</span> PHASE1, PHASE2, PHASE3, PHASE4<a name="line.66"></a>
75 <span class="sourceLineNo">067</span> }<a name="line.67"></a>
76 <span class="sourceLineNo">068</span><a name="line.68"></a>
77 <span class="sourceLineNo">069</span> public final static String FAILURE_PHASE_KEY = "failurePhase";<a name="line.69"></a>
78 <span class="sourceLineNo">070</span><a name="line.70"></a>
79 <span class="sourceLineNo">071</span> static class BackupMergeJobWithFailures extends MapReduceBackupMergeJob {<a name="line.71"></a>
80 <span class="sourceLineNo">072</span> FailurePhase failurePhase;<a name="line.72"></a>
81 <span class="sourceLineNo">073</span><a name="line.73"></a>
82 <span class="sourceLineNo">074</span> @Override<a name="line.74"></a>
83 <span class="sourceLineNo">075</span> public void setConf(Configuration conf) {<a name="line.75"></a>
84 <span class="sourceLineNo">076</span> super.setConf(conf);<a name="line.76"></a>
85 <span class="sourceLineNo">077</span> String val = conf.get(FAILURE_PHASE_KEY);<a name="line.77"></a>
86 <span class="sourceLineNo">078</span> if (val != null) {<a name="line.78"></a>
87 <span class="sourceLineNo">079</span> failurePhase = FailurePhase.valueOf(val);<a name="line.79"></a>
88 <span class="sourceLineNo">080</span> } else {<a name="line.80"></a>
89 <span class="sourceLineNo">081</span> Assert.fail("Failure phase is not set");<a name="line.81"></a>
90 <span class="sourceLineNo">082</span> }<a name="line.82"></a>
91 <span class="sourceLineNo">083</span> }<a name="line.83"></a>
92 <span class="sourceLineNo">084</span><a name="line.84"></a>
93 <span class="sourceLineNo">085</span> /**<a name="line.85"></a>
94 <span class="sourceLineNo">086</span> * This is the exact copy of parent's run() with injections<a name="line.86"></a>
95 <span class="sourceLineNo">087</span> * of different types of failures<a name="line.87"></a>
96 <span class="sourceLineNo">088</span> */<a name="line.88"></a>
97 <span class="sourceLineNo">089</span> @Override<a name="line.89"></a>
98 <span class="sourceLineNo">090</span> public void run(String[] backupIds) throws IOException {<a name="line.90"></a>
99 <span class="sourceLineNo">091</span> String bulkOutputConfKey;<a name="line.91"></a>
100 <span class="sourceLineNo">092</span><a name="line.92"></a>
101 <span class="sourceLineNo">093</span> // TODO : run player on remote cluster<a name="line.93"></a>
102 <span class="sourceLineNo">094</span> player = new MapReduceHFileSplitterJob();<a name="line.94"></a>
103 <span class="sourceLineNo">095</span> bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY;<a name="line.95"></a>
104 <span class="sourceLineNo">096</span> // Player reads all files in arbitrary directory structure and creates<a name="line.96"></a>
105 <span class="sourceLineNo">097</span> // a Map task for each file<a name="line.97"></a>
106 <span class="sourceLineNo">098</span> String bids = StringUtils.join(backupIds, ",");<a name="line.98"></a>
107 <span class="sourceLineNo">099</span><a name="line.99"></a>
108 <span class="sourceLineNo">100</span> if (LOG.isDebugEnabled()) {<a name="line.100"></a>
109 <span class="sourceLineNo">101</span> LOG.debug("Merge backup images " + bids);<a name="line.101"></a>
110 <span class="sourceLineNo">102</span> }<a name="line.102"></a>
111 <span class="sourceLineNo">103</span><a name="line.103"></a>
112 <span class="sourceLineNo">104</span> List&lt;Pair&lt;TableName, Path&gt;&gt; processedTableList = new ArrayList&lt;&gt;();<a name="line.104"></a>
113 <span class="sourceLineNo">105</span> boolean finishedTables = false;<a name="line.105"></a>
114 <span class="sourceLineNo">106</span> Connection conn = ConnectionFactory.createConnection(getConf());<a name="line.106"></a>
115 <span class="sourceLineNo">107</span> BackupSystemTable table = new BackupSystemTable(conn);<a name="line.107"></a>
116 <span class="sourceLineNo">108</span> FileSystem fs = FileSystem.get(getConf());<a name="line.108"></a>
117 <span class="sourceLineNo">109</span><a name="line.109"></a>
118 <span class="sourceLineNo">110</span> try {<a name="line.110"></a>
119 <span class="sourceLineNo">111</span> // Start backup exclusive operation<a name="line.111"></a>
120 <span class="sourceLineNo">112</span> table.startBackupExclusiveOperation();<a name="line.112"></a>
121 <span class="sourceLineNo">113</span> // Start merge operation<a name="line.113"></a>
122 <span class="sourceLineNo">114</span> table.startMergeOperation(backupIds);<a name="line.114"></a>
123 <span class="sourceLineNo">115</span><a name="line.115"></a>
124 <span class="sourceLineNo">116</span> // Select most recent backup id<a name="line.116"></a>
125 <span class="sourceLineNo">117</span> String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);<a name="line.117"></a>
126 <span class="sourceLineNo">118</span><a name="line.118"></a>
127 <span class="sourceLineNo">119</span> TableName[] tableNames = getTableNamesInBackupImages(backupIds);<a name="line.119"></a>
128 <span class="sourceLineNo">120</span><a name="line.120"></a>
129 <span class="sourceLineNo">121</span> BackupInfo bInfo = table.readBackupInfo(backupIds[0]);<a name="line.121"></a>
130 <span class="sourceLineNo">122</span> String backupRoot = bInfo.getBackupRootDir();<a name="line.122"></a>
131 <span class="sourceLineNo">123</span> // PHASE 1<a name="line.123"></a>
132 <span class="sourceLineNo">124</span> checkFailure(FailurePhase.PHASE1);<a name="line.124"></a>
133 <span class="sourceLineNo">125</span><a name="line.125"></a>
134 <span class="sourceLineNo">126</span> for (int i = 0; i &lt; tableNames.length; i++) {<a name="line.126"></a>
135 <span class="sourceLineNo">127</span> LOG.info("Merge backup images for " + tableNames[i]);<a name="line.127"></a>
136 <span class="sourceLineNo">128</span><a name="line.128"></a>
137 <span class="sourceLineNo">129</span> // Find input directories for table<a name="line.129"></a>
138 <span class="sourceLineNo">130</span> Path[] dirPaths = findInputDirectories(fs, backupRoot, tableNames[i], backupIds);<a name="line.130"></a>
139 <span class="sourceLineNo">131</span> String dirs = StringUtils.join(dirPaths, ",");<a name="line.131"></a>
140 <span class="sourceLineNo">132</span> Path bulkOutputPath =<a name="line.132"></a>
141 <span class="sourceLineNo">133</span> BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(tableNames[i]),<a name="line.133"></a>
142 <span class="sourceLineNo">134</span> getConf(), false);<a name="line.134"></a>
143 <span class="sourceLineNo">135</span> // Delete content if exists<a name="line.135"></a>
144 <span class="sourceLineNo">136</span> if (fs.exists(bulkOutputPath)) {<a name="line.136"></a>
145 <span class="sourceLineNo">137</span> if (!fs.delete(bulkOutputPath, true)) {<a name="line.137"></a>
146 <span class="sourceLineNo">138</span> LOG.warn("Can not delete: " + bulkOutputPath);<a name="line.138"></a>
147 <span class="sourceLineNo">139</span> }<a name="line.139"></a>
148 <span class="sourceLineNo">140</span> }<a name="line.140"></a>
149 <span class="sourceLineNo">141</span> Configuration conf = getConf();<a name="line.141"></a>
150 <span class="sourceLineNo">142</span> conf.set(bulkOutputConfKey, bulkOutputPath.toString());<a name="line.142"></a>
151 <span class="sourceLineNo">143</span> String[] playerArgs = { dirs, tableNames[i].getNameAsString() };<a name="line.143"></a>
152 <span class="sourceLineNo">144</span><a name="line.144"></a>
153 <span class="sourceLineNo">145</span> // PHASE 2<a name="line.145"></a>
154 <span class="sourceLineNo">146</span> checkFailure(FailurePhase.PHASE2);<a name="line.146"></a>
155 <span class="sourceLineNo">147</span> player.setConf(getConf());<a name="line.147"></a>
156 <span class="sourceLineNo">148</span> int result = player.run(playerArgs);<a name="line.148"></a>
157 <span class="sourceLineNo">149</span> if (succeeded(result)) {<a name="line.149"></a>
158 <span class="sourceLineNo">150</span> // Add to processed table list<a name="line.150"></a>
159 <span class="sourceLineNo">151</span> processedTableList.add(new Pair&lt;&gt;(tableNames[i], bulkOutputPath));<a name="line.151"></a>
160 <span class="sourceLineNo">152</span> } else {<a name="line.152"></a>
161 <span class="sourceLineNo">153</span> throw new IOException("Can not merge backup images for " + dirs<a name="line.153"></a>
162 <span class="sourceLineNo">154</span> + " (check Hadoop/MR and HBase logs). Player return code =" + result);<a name="line.154"></a>
163 <span class="sourceLineNo">155</span> }<a name="line.155"></a>
164 <span class="sourceLineNo">156</span> LOG.debug("Merge Job finished:" + result);<a name="line.156"></a>
165 <span class="sourceLineNo">157</span> }<a name="line.157"></a>
166 <span class="sourceLineNo">158</span> List&lt;TableName&gt; tableList = toTableNameList(processedTableList);<a name="line.158"></a>
167 <span class="sourceLineNo">159</span> // PHASE 3<a name="line.159"></a>
168 <span class="sourceLineNo">160</span> checkFailure(FailurePhase.PHASE3);<a name="line.160"></a>
169 <span class="sourceLineNo">161</span> table.updateProcessedTablesForMerge(tableList);<a name="line.161"></a>
170 <span class="sourceLineNo">162</span> finishedTables = true;<a name="line.162"></a>
171 <span class="sourceLineNo">163</span><a name="line.163"></a>
172 <span class="sourceLineNo">164</span> // (modification of a backup file system)<a name="line.164"></a>
173 <span class="sourceLineNo">165</span> // Move existing mergedBackupId data into tmp directory<a name="line.165"></a>
174 <span class="sourceLineNo">166</span> // we will need it later in case of a failure<a name="line.166"></a>
175 <span class="sourceLineNo">167</span> Path tmpBackupDir = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot,<a name="line.167"></a>
176 <span class="sourceLineNo">168</span> mergedBackupId);<a name="line.168"></a>
177 <span class="sourceLineNo">169</span> Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, mergedBackupId);<a name="line.169"></a>
178 <span class="sourceLineNo">170</span> if (!fs.rename(backupDirPath, tmpBackupDir)) {<a name="line.170"></a>
179 <span class="sourceLineNo">171</span> throw new IOException("Failed to rename "+ backupDirPath +" to "+tmpBackupDir);<a name="line.171"></a>
180 <span class="sourceLineNo">172</span> } else {<a name="line.172"></a>
181 <span class="sourceLineNo">173</span> LOG.debug("Renamed "+ backupDirPath +" to "+ tmpBackupDir);<a name="line.173"></a>
182 <span class="sourceLineNo">174</span> }<a name="line.174"></a>
183 <span class="sourceLineNo">175</span> // Move new data into backup dest<a name="line.175"></a>
184 <span class="sourceLineNo">176</span> for (Pair&lt;TableName, Path&gt; tn : processedTableList) {<a name="line.176"></a>
185 <span class="sourceLineNo">177</span> moveData(fs, backupRoot, tn.getSecond(), tn.getFirst(), mergedBackupId);<a name="line.177"></a>
186 <span class="sourceLineNo">178</span> }<a name="line.178"></a>
187 <span class="sourceLineNo">179</span> checkFailure(FailurePhase.PHASE4);<a name="line.179"></a>
188 <span class="sourceLineNo">180</span> // Update backup manifest<a name="line.180"></a>
189 <span class="sourceLineNo">181</span> List&lt;String&gt; backupsToDelete = getBackupIdsToDelete(backupIds, mergedBackupId);<a name="line.181"></a>
190 <span class="sourceLineNo">182</span> updateBackupManifest(tmpBackupDir.getParent().toString(), mergedBackupId, backupsToDelete);<a name="line.182"></a>
191 <span class="sourceLineNo">183</span> // Copy meta files back from tmp to backup dir<a name="line.183"></a>
192 <span class="sourceLineNo">184</span> copyMetaData(fs, tmpBackupDir, backupDirPath);<a name="line.184"></a>
193 <span class="sourceLineNo">185</span> // Delete tmp dir (Rename back during repair)<a name="line.185"></a>
194 <span class="sourceLineNo">186</span> if (!fs.delete(tmpBackupDir, true)) {<a name="line.186"></a>
195 <span class="sourceLineNo">187</span> // WARN and ignore<a name="line.187"></a>
196 <span class="sourceLineNo">188</span> LOG.warn("Could not delete tmp dir: "+ tmpBackupDir);<a name="line.188"></a>
197 <span class="sourceLineNo">189</span> }<a name="line.189"></a>
198 <span class="sourceLineNo">190</span> // Delete old data<a name="line.190"></a>
199 <span class="sourceLineNo">191</span> deleteBackupImages(backupsToDelete, conn, fs, backupRoot);<a name="line.191"></a>
200 <span class="sourceLineNo">192</span> // Finish merge session<a name="line.192"></a>
201 <span class="sourceLineNo">193</span> table.finishMergeOperation();<a name="line.193"></a>
202 <span class="sourceLineNo">194</span> // Release lock<a name="line.194"></a>
203 <span class="sourceLineNo">195</span> table.finishBackupExclusiveOperation();<a name="line.195"></a>
204 <span class="sourceLineNo">196</span> } catch (RuntimeException e) {<a name="line.196"></a>
205 <span class="sourceLineNo">197</span> throw e;<a name="line.197"></a>
206 <span class="sourceLineNo">198</span> } catch (Exception e) {<a name="line.198"></a>
207 <span class="sourceLineNo">199</span> LOG.error(e.toString(), e);<a name="line.199"></a>
208 <span class="sourceLineNo">200</span> if (!finishedTables) {<a name="line.200"></a>
209 <span class="sourceLineNo">201</span> // cleanup bulk directories and finish merge<a name="line.201"></a>
210 <span class="sourceLineNo">202</span> // merge MUST be repeated (no need for repair)<a name="line.202"></a>
211 <span class="sourceLineNo">203</span> cleanupBulkLoadDirs(fs, toPathList(processedTableList));<a name="line.203"></a>
212 <span class="sourceLineNo">204</span> table.finishMergeOperation();<a name="line.204"></a>
213 <span class="sourceLineNo">205</span> table.finishBackupExclusiveOperation();<a name="line.205"></a>
214 <span class="sourceLineNo">206</span> throw new IOException("Backup merge operation failed, you should try it again", e);<a name="line.206"></a>
215 <span class="sourceLineNo">207</span> } else {<a name="line.207"></a>
216 <span class="sourceLineNo">208</span> // backup repair must be run<a name="line.208"></a>
217 <span class="sourceLineNo">209</span> throw new IOException(<a name="line.209"></a>
218 <span class="sourceLineNo">210</span> "Backup merge operation failed, run backup repair tool to restore system's integrity",<a name="line.210"></a>
219 <span class="sourceLineNo">211</span> e);<a name="line.211"></a>
220 <span class="sourceLineNo">212</span> }<a name="line.212"></a>
221 <span class="sourceLineNo">213</span> } finally {<a name="line.213"></a>
222 <span class="sourceLineNo">214</span> table.close();<a name="line.214"></a>
223 <span class="sourceLineNo">215</span> conn.close();<a name="line.215"></a>
224 <span class="sourceLineNo">216</span> }<a name="line.216"></a>
225 <span class="sourceLineNo">217</span> }<a name="line.217"></a>
226 <span class="sourceLineNo">218</span><a name="line.218"></a>
227 <span class="sourceLineNo">219</span> private void checkFailure(FailurePhase phase) throws IOException {<a name="line.219"></a>
228 <span class="sourceLineNo">220</span> if (failurePhase != null &amp;&amp; failurePhase == phase) {<a name="line.220"></a>
229 <span class="sourceLineNo">221</span> throw new IOException(phase.toString());<a name="line.221"></a>
230 <span class="sourceLineNo">222</span> }<a name="line.222"></a>
231 <span class="sourceLineNo">223</span> }<a name="line.223"></a>
232 <span class="sourceLineNo">224</span> }<a name="line.224"></a>
233 <span class="sourceLineNo">225</span><a name="line.225"></a>
234 <span class="sourceLineNo">226</span> @Test<a name="line.226"></a>
235 <span class="sourceLineNo">227</span> public void TestIncBackupMergeRestore() throws Exception {<a name="line.227"></a>
236 <span class="sourceLineNo">228</span> int ADD_ROWS = 99;<a name="line.228"></a>
237 <span class="sourceLineNo">229</span> // #1 - create full backup for all tables<a name="line.229"></a>
238 <span class="sourceLineNo">230</span> LOG.info("create full backup image for all tables");<a name="line.230"></a>
239 <span class="sourceLineNo">231</span><a name="line.231"></a>
240 <span class="sourceLineNo">232</span> List&lt;TableName&gt; tables = Lists.newArrayList(table1, table2);<a name="line.232"></a>
241 <span class="sourceLineNo">233</span> // Set custom Merge Job implementation<a name="line.233"></a>
242 <span class="sourceLineNo">234</span> conf1.setClass(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS,<a name="line.234"></a>
243 <span class="sourceLineNo">235</span> BackupMergeJobWithFailures.class, BackupMergeJob.class);<a name="line.235"></a>
244 <span class="sourceLineNo">236</span><a name="line.236"></a>
245 <span class="sourceLineNo">237</span> Connection conn = ConnectionFactory.createConnection(conf1);<a name="line.237"></a>
246 <span class="sourceLineNo">238</span><a name="line.238"></a>
247 <span class="sourceLineNo">239</span> HBaseAdmin admin = (HBaseAdmin) conn.getAdmin();<a name="line.239"></a>
248 <span class="sourceLineNo">240</span> BackupAdminImpl client = new BackupAdminImpl(conn);<a name="line.240"></a>
249 <span class="sourceLineNo">241</span><a name="line.241"></a>
250 <span class="sourceLineNo">242</span> BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);<a name="line.242"></a>
251 <span class="sourceLineNo">243</span> String backupIdFull = client.backupTables(request);<a name="line.243"></a>
252 <span class="sourceLineNo">244</span><a name="line.244"></a>
253 <span class="sourceLineNo">245</span> assertTrue(checkSucceeded(backupIdFull));<a name="line.245"></a>
254 <span class="sourceLineNo">246</span><a name="line.246"></a>
255 <span class="sourceLineNo">247</span> // #2 - insert some data to table1<a name="line.247"></a>
256 <span class="sourceLineNo">248</span> HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);<a name="line.248"></a>
257 <span class="sourceLineNo">249</span> LOG.debug("writing " + ADD_ROWS + " rows to " + table1);<a name="line.249"></a>
258 <span class="sourceLineNo">250</span><a name="line.250"></a>
259 <span class="sourceLineNo">251</span> Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS);<a name="line.251"></a>
260 <span class="sourceLineNo">252</span> t1.close();<a name="line.252"></a>
261 <span class="sourceLineNo">253</span> LOG.debug("written " + ADD_ROWS + " rows to " + table1);<a name="line.253"></a>
262 <span class="sourceLineNo">254</span><a name="line.254"></a>
263 <span class="sourceLineNo">255</span> HTable t2 = insertIntoTable(conn, table2, famName, 1, ADD_ROWS);<a name="line.255"></a>
264 <span class="sourceLineNo">256</span><a name="line.256"></a>
265 <span class="sourceLineNo">257</span> Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + ADD_ROWS);<a name="line.257"></a>
266 <span class="sourceLineNo">258</span> t2.close();<a name="line.258"></a>
267 <span class="sourceLineNo">259</span> LOG.debug("written " + ADD_ROWS + " rows to " + table2);<a name="line.259"></a>
268 <span class="sourceLineNo">260</span><a name="line.260"></a>
269 <span class="sourceLineNo">261</span> // #3 - incremental backup for multiple tables<a name="line.261"></a>
270 <span class="sourceLineNo">262</span> tables = Lists.newArrayList(table1, table2);<a name="line.262"></a>
271 <span class="sourceLineNo">263</span> request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);<a name="line.263"></a>
272 <span class="sourceLineNo">264</span> String backupIdIncMultiple = client.backupTables(request);<a name="line.264"></a>
273 <span class="sourceLineNo">265</span><a name="line.265"></a>
274 <span class="sourceLineNo">266</span> assertTrue(checkSucceeded(backupIdIncMultiple));<a name="line.266"></a>
275 <span class="sourceLineNo">267</span><a name="line.267"></a>
276 <span class="sourceLineNo">268</span> t1 = insertIntoTable(conn, table1, famName, 2, ADD_ROWS);<a name="line.268"></a>
277 <span class="sourceLineNo">269</span> t1.close();<a name="line.269"></a>
278 <span class="sourceLineNo">270</span><a name="line.270"></a>
279 <span class="sourceLineNo">271</span> t2 = insertIntoTable(conn, table2, famName, 2, ADD_ROWS);<a name="line.271"></a>
280 <span class="sourceLineNo">272</span> t2.close();<a name="line.272"></a>
281 <span class="sourceLineNo">273</span><a name="line.273"></a>
282 <span class="sourceLineNo">274</span> // #3 - incremental backup for multiple tables<a name="line.274"></a>
283 <span class="sourceLineNo">275</span> request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);<a name="line.275"></a>
284 <span class="sourceLineNo">276</span> String backupIdIncMultiple2 = client.backupTables(request);<a name="line.276"></a>
285 <span class="sourceLineNo">277</span> assertTrue(checkSucceeded(backupIdIncMultiple2));<a name="line.277"></a>
286 <span class="sourceLineNo">278</span> // #4 Merge backup images with failures<a name="line.278"></a>
287 <span class="sourceLineNo">279</span><a name="line.279"></a>
288 <span class="sourceLineNo">280</span> for (FailurePhase phase : FailurePhase.values()) {<a name="line.280"></a>
289 <span class="sourceLineNo">281</span> Configuration conf = conn.getConfiguration();<a name="line.281"></a>
290 <span class="sourceLineNo">282</span><a name="line.282"></a>
291 <span class="sourceLineNo">283</span> conf.set(FAILURE_PHASE_KEY, phase.toString());<a name="line.283"></a>
292 <span class="sourceLineNo">284</span><a name="line.284"></a>
293 <span class="sourceLineNo">285</span> try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {<a name="line.285"></a>
294 <span class="sourceLineNo">286</span> String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };<a name="line.286"></a>
295 <span class="sourceLineNo">287</span> bAdmin.mergeBackups(backups);<a name="line.287"></a>
296 <span class="sourceLineNo">288</span> Assert.fail("Expected IOException");<a name="line.288"></a>
297 <span class="sourceLineNo">289</span> } catch (IOException e) {<a name="line.289"></a>
298 <span class="sourceLineNo">290</span> BackupSystemTable table = new BackupSystemTable(conn);<a name="line.290"></a>
299 <span class="sourceLineNo">291</span> if(phase.ordinal() &lt; FailurePhase.PHASE4.ordinal()) {<a name="line.291"></a>
300 <span class="sourceLineNo">292</span> // No need to repair:<a name="line.292"></a>
301 <span class="sourceLineNo">293</span> // Both Merge and backup exclusive operations are finished<a name="line.293"></a>
302 <span class="sourceLineNo">294</span> assertFalse(table.isMergeInProgress());<a name="line.294"></a>
303 <span class="sourceLineNo">295</span> try {<a name="line.295"></a>
304 <span class="sourceLineNo">296</span> table.finishBackupExclusiveOperation();<a name="line.296"></a>
305 <span class="sourceLineNo">297</span> Assert.fail("IOException is expected");<a name="line.297"></a>
306 <span class="sourceLineNo">298</span> } catch(IOException ee) {<a name="line.298"></a>
307 <span class="sourceLineNo">299</span> // Expected<a name="line.299"></a>
308 <span class="sourceLineNo">300</span> }<a name="line.300"></a>
309 <span class="sourceLineNo">301</span> } else {<a name="line.301"></a>
310 <span class="sourceLineNo">302</span> // Repair is required<a name="line.302"></a>
311 <span class="sourceLineNo">303</span> assertTrue(table.isMergeInProgress());<a name="line.303"></a>
312 <span class="sourceLineNo">304</span> try {<a name="line.304"></a>
313 <span class="sourceLineNo">305</span> table.startBackupExclusiveOperation();<a name="line.305"></a>
314 <span class="sourceLineNo">306</span> Assert.fail("IOException is expected");<a name="line.306"></a>
315 <span class="sourceLineNo">307</span> } catch(IOException ee) {<a name="line.307"></a>
316 <span class="sourceLineNo">308</span> // Expected - clean up before proceeding<a name="line.308"></a>
317 <span class="sourceLineNo">309</span> //table.finishMergeOperation();<a name="line.309"></a>
318 <span class="sourceLineNo">310</span> //table.finishBackupExclusiveOperation();<a name="line.310"></a>
319 <span class="sourceLineNo">311</span> }<a name="line.311"></a>
320 <span class="sourceLineNo">312</span> }<a name="line.312"></a>
321 <span class="sourceLineNo">313</span> table.close();<a name="line.313"></a>
322 <span class="sourceLineNo">314</span> LOG.debug("Expected :"+ e.getMessage());<a name="line.314"></a>
323 <span class="sourceLineNo">315</span> }<a name="line.315"></a>
324 <span class="sourceLineNo">316</span> }<a name="line.316"></a>
325 <span class="sourceLineNo">317</span> // Now merge w/o failures<a name="line.317"></a>
326 <span class="sourceLineNo">318</span> Configuration conf = conn.getConfiguration();<a name="line.318"></a>
327 <span class="sourceLineNo">319</span> conf.unset(FAILURE_PHASE_KEY);<a name="line.319"></a>
328 <span class="sourceLineNo">320</span> conf.unset(BackupRestoreFactory.HBASE_BACKUP_MERGE_IMPL_CLASS);<a name="line.320"></a>
329 <span class="sourceLineNo">321</span> // Now run repair<a name="line.321"></a>
330 <span class="sourceLineNo">322</span> BackupSystemTable sysTable = new BackupSystemTable(conn);<a name="line.322"></a>
331 <span class="sourceLineNo">323</span> BackupCommands.RepairCommand.repairFailedBackupMergeIfAny(conn, sysTable);<a name="line.323"></a>
332 <span class="sourceLineNo">324</span> // Now repeat merge<a name="line.324"></a>
333 <span class="sourceLineNo">325</span> try (BackupAdmin bAdmin = new BackupAdminImpl(conn)) {<a name="line.325"></a>
334 <span class="sourceLineNo">326</span> String[] backups = new String[] { backupIdIncMultiple, backupIdIncMultiple2 };<a name="line.326"></a>
335 <span class="sourceLineNo">327</span> bAdmin.mergeBackups(backups);<a name="line.327"></a>
336 <span class="sourceLineNo">328</span> }<a name="line.328"></a>
337 <span class="sourceLineNo">329</span><a name="line.329"></a>
338 <span class="sourceLineNo">330</span> // #6 - restore incremental backup for multiple tables, with overwrite<a name="line.330"></a>
339 <span class="sourceLineNo">331</span> TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };<a name="line.331"></a>
340 <span class="sourceLineNo">332</span> TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };<a name="line.332"></a>
341 <span class="sourceLineNo">333</span> client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2, false,<a name="line.333"></a>
342 <span class="sourceLineNo">334</span> tablesRestoreIncMultiple, tablesMapIncMultiple, true));<a name="line.334"></a>
343 <span class="sourceLineNo">335</span><a name="line.335"></a>
344 <span class="sourceLineNo">336</span> Table hTable = conn.getTable(table1_restore);<a name="line.336"></a>
345 <span class="sourceLineNo">337</span> LOG.debug("After incremental restore: " + hTable.getTableDescriptor());<a name="line.337"></a>
346 <span class="sourceLineNo">338</span> LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows");<a name="line.338"></a>
347 <span class="sourceLineNo">339</span> Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + 2 * ADD_ROWS);<a name="line.339"></a>
348 <span class="sourceLineNo">340</span><a name="line.340"></a>
349 <span class="sourceLineNo">341</span> hTable.close();<a name="line.341"></a>
350 <span class="sourceLineNo">342</span><a name="line.342"></a>
351 <span class="sourceLineNo">343</span> hTable = conn.getTable(table2_restore);<a name="line.343"></a>
352 <span class="sourceLineNo">344</span> Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 2 * ADD_ROWS);<a name="line.344"></a>
353 <span class="sourceLineNo">345</span> hTable.close();<a name="line.345"></a>
354 <span class="sourceLineNo">346</span><a name="line.346"></a>
355 <span class="sourceLineNo">347</span> admin.close();<a name="line.347"></a>
356 <span class="sourceLineNo">348</span> conn.close();<a name="line.348"></a>
357 <span class="sourceLineNo">349</span> }<a name="line.349"></a>
358 <span class="sourceLineNo">350</span>}<a name="line.350"></a>
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419 </pre>
420 </div>
421 </body>
422 </html>