Published site at d5aaeee88b331e064830a2774f4fed238631457c.
[hbase-site.git] / devapidocs / src-html / org / apache / hadoop / hbase / backup / mapreduce / MapReduceBackupMergeJob.html
1 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
2 <html lang="en">
3 <head>
4 <title>Source code</title>
5 <link rel="stylesheet" type="text/css" href="../../../../../../../stylesheet.css" title="Style">
6 </head>
7 <body>
8 <div class="sourceContainer">
9 <pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a>
10 <span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one<a name="line.2"></a>
11 <span class="sourceLineNo">003</span> * or more contributor license agreements. See the NOTICE file<a name="line.3"></a>
12 <span class="sourceLineNo">004</span> * distributed with this work for additional information<a name="line.4"></a>
13 <span class="sourceLineNo">005</span> * regarding copyright ownership. The ASF licenses this file<a name="line.5"></a>
14 <span class="sourceLineNo">006</span> * to you under the Apache License, Version 2.0 (the<a name="line.6"></a>
15 <span class="sourceLineNo">007</span> * "License"); you may not use this file except in compliance<a name="line.7"></a>
16 <span class="sourceLineNo">008</span> * with the License. You may obtain a copy of the License at<a name="line.8"></a>
17 <span class="sourceLineNo">009</span> *<a name="line.9"></a>
18 <span class="sourceLineNo">010</span> * http://www.apache.org/licenses/LICENSE-2.0<a name="line.10"></a>
19 <span class="sourceLineNo">011</span> *<a name="line.11"></a>
20 <span class="sourceLineNo">012</span> * Unless required by applicable law or agreed to in writing, software<a name="line.12"></a>
21 <span class="sourceLineNo">013</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.13"></a>
22 <span class="sourceLineNo">014</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.14"></a>
23 <span class="sourceLineNo">015</span> * See the License for the specific language governing permissions and<a name="line.15"></a>
24 <span class="sourceLineNo">016</span> * limitations under the License.<a name="line.16"></a>
25 <span class="sourceLineNo">017</span> */<a name="line.17"></a>
26 <span class="sourceLineNo">018</span>package org.apache.hadoop.hbase.backup.mapreduce;<a name="line.18"></a>
27 <span class="sourceLineNo">019</span><a name="line.19"></a>
28 <span class="sourceLineNo">020</span>import static org.apache.hadoop.hbase.backup.util.BackupUtils.succeeded;<a name="line.20"></a>
29 <span class="sourceLineNo">021</span><a name="line.21"></a>
30 <span class="sourceLineNo">022</span>import java.io.File;<a name="line.22"></a>
31 <span class="sourceLineNo">023</span>import java.io.IOException;<a name="line.23"></a>
32 <span class="sourceLineNo">024</span>import java.util.ArrayList;<a name="line.24"></a>
33 <span class="sourceLineNo">025</span>import java.util.HashSet;<a name="line.25"></a>
34 <span class="sourceLineNo">026</span>import java.util.List;<a name="line.26"></a>
35 <span class="sourceLineNo">027</span>import java.util.Set;<a name="line.27"></a>
36 <span class="sourceLineNo">028</span>import java.util.Stack;<a name="line.28"></a>
37 <span class="sourceLineNo">029</span><a name="line.29"></a>
38 <span class="sourceLineNo">030</span>import org.apache.commons.lang3.StringUtils;<a name="line.30"></a>
39 <span class="sourceLineNo">031</span>import org.apache.hadoop.conf.Configuration;<a name="line.31"></a>
40 <span class="sourceLineNo">032</span>import org.apache.hadoop.fs.FileStatus;<a name="line.32"></a>
41 <span class="sourceLineNo">033</span>import org.apache.hadoop.fs.FileSystem;<a name="line.33"></a>
42 <span class="sourceLineNo">034</span>import org.apache.hadoop.fs.LocatedFileStatus;<a name="line.34"></a>
43 <span class="sourceLineNo">035</span>import org.apache.hadoop.fs.Path;<a name="line.35"></a>
44 <span class="sourceLineNo">036</span>import org.apache.hadoop.fs.RemoteIterator;<a name="line.36"></a>
45 <span class="sourceLineNo">037</span>import org.apache.hadoop.hbase.TableName;<a name="line.37"></a>
46 <span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.backup.BackupInfo;<a name="line.38"></a>
47 <span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.backup.BackupMergeJob;<a name="line.39"></a>
48 <span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.backup.HBackupFileSystem;<a name="line.40"></a>
49 <span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.backup.impl.BackupManifest;<a name="line.41"></a>
50 <span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;<a name="line.42"></a>
51 <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.backup.util.BackupUtils;<a name="line.43"></a>
52 <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.44"></a>
53 <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.client.ConnectionFactory;<a name="line.45"></a>
54 <span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;<a name="line.46"></a>
55 <span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.util.FSTableDescriptors;<a name="line.47"></a>
56 <span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.48"></a>
57 <span class="sourceLineNo">049</span>import org.apache.hadoop.util.Tool;<a name="line.49"></a>
58 <span class="sourceLineNo">050</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.50"></a>
59 <span class="sourceLineNo">051</span>import org.slf4j.Logger;<a name="line.51"></a>
60 <span class="sourceLineNo">052</span>import org.slf4j.LoggerFactory;<a name="line.52"></a>
61 <span class="sourceLineNo">053</span><a name="line.53"></a>
62 <span class="sourceLineNo">054</span>/**<a name="line.54"></a>
63 <span class="sourceLineNo">055</span> * MapReduce implementation of {@link BackupMergeJob}<a name="line.55"></a>
64 <span class="sourceLineNo">056</span> * Must be initialized with configuration of a backup destination cluster<a name="line.56"></a>
65 <span class="sourceLineNo">057</span> *<a name="line.57"></a>
66 <span class="sourceLineNo">058</span> */<a name="line.58"></a>
67 <span class="sourceLineNo">059</span>@InterfaceAudience.Private<a name="line.59"></a>
68 <span class="sourceLineNo">060</span>public class MapReduceBackupMergeJob implements BackupMergeJob {<a name="line.60"></a>
69 <span class="sourceLineNo">061</span> public static final Logger LOG = LoggerFactory.getLogger(MapReduceBackupMergeJob.class);<a name="line.61"></a>
70 <span class="sourceLineNo">062</span><a name="line.62"></a>
71 <span class="sourceLineNo">063</span> protected Tool player;<a name="line.63"></a>
72 <span class="sourceLineNo">064</span> protected Configuration conf;<a name="line.64"></a>
73 <span class="sourceLineNo">065</span><a name="line.65"></a>
74 <span class="sourceLineNo">066</span> public MapReduceBackupMergeJob() {<a name="line.66"></a>
75 <span class="sourceLineNo">067</span> }<a name="line.67"></a>
76 <span class="sourceLineNo">068</span><a name="line.68"></a>
77 <span class="sourceLineNo">069</span> @Override<a name="line.69"></a>
78 <span class="sourceLineNo">070</span> public Configuration getConf() {<a name="line.70"></a>
79 <span class="sourceLineNo">071</span> return conf;<a name="line.71"></a>
80 <span class="sourceLineNo">072</span> }<a name="line.72"></a>
81 <span class="sourceLineNo">073</span><a name="line.73"></a>
82 <span class="sourceLineNo">074</span> @Override<a name="line.74"></a>
83 <span class="sourceLineNo">075</span> public void setConf(Configuration conf) {<a name="line.75"></a>
84 <span class="sourceLineNo">076</span> this.conf = conf;<a name="line.76"></a>
85 <span class="sourceLineNo">077</span> }<a name="line.77"></a>
86 <span class="sourceLineNo">078</span><a name="line.78"></a>
87 <span class="sourceLineNo">079</span> @Override<a name="line.79"></a>
88 <span class="sourceLineNo">080</span> public void run(String[] backupIds) throws IOException {<a name="line.80"></a>
89 <span class="sourceLineNo">081</span> String bulkOutputConfKey;<a name="line.81"></a>
90 <span class="sourceLineNo">082</span><a name="line.82"></a>
91 <span class="sourceLineNo">083</span> // TODO : run player on remote cluster<a name="line.83"></a>
92 <span class="sourceLineNo">084</span> player = new MapReduceHFileSplitterJob();<a name="line.84"></a>
93 <span class="sourceLineNo">085</span> bulkOutputConfKey = MapReduceHFileSplitterJob.BULK_OUTPUT_CONF_KEY;<a name="line.85"></a>
94 <span class="sourceLineNo">086</span> // Player reads all files in arbitrary directory structure and creates<a name="line.86"></a>
95 <span class="sourceLineNo">087</span> // a Map task for each file<a name="line.87"></a>
96 <span class="sourceLineNo">088</span> String bids = StringUtils.join(backupIds, ",");<a name="line.88"></a>
97 <span class="sourceLineNo">089</span><a name="line.89"></a>
98 <span class="sourceLineNo">090</span> if (LOG.isDebugEnabled()) {<a name="line.90"></a>
99 <span class="sourceLineNo">091</span> LOG.debug("Merge backup images " + bids);<a name="line.91"></a>
100 <span class="sourceLineNo">092</span> }<a name="line.92"></a>
101 <span class="sourceLineNo">093</span><a name="line.93"></a>
102 <span class="sourceLineNo">094</span> List&lt;Pair&lt;TableName, Path&gt;&gt; processedTableList = new ArrayList&lt;&gt;();<a name="line.94"></a>
103 <span class="sourceLineNo">095</span> boolean finishedTables = false;<a name="line.95"></a>
104 <span class="sourceLineNo">096</span> Connection conn = ConnectionFactory.createConnection(getConf());<a name="line.96"></a>
105 <span class="sourceLineNo">097</span> BackupSystemTable table = new BackupSystemTable(conn);<a name="line.97"></a>
106 <span class="sourceLineNo">098</span> FileSystem fs = FileSystem.get(getConf());<a name="line.98"></a>
107 <span class="sourceLineNo">099</span><a name="line.99"></a>
108 <span class="sourceLineNo">100</span> try {<a name="line.100"></a>
109 <span class="sourceLineNo">101</span><a name="line.101"></a>
110 <span class="sourceLineNo">102</span> // Get exclusive lock on backup system<a name="line.102"></a>
111 <span class="sourceLineNo">103</span> table.startBackupExclusiveOperation();<a name="line.103"></a>
112 <span class="sourceLineNo">104</span> // Start merge operation<a name="line.104"></a>
113 <span class="sourceLineNo">105</span> table.startMergeOperation(backupIds);<a name="line.105"></a>
114 <span class="sourceLineNo">106</span><a name="line.106"></a>
115 <span class="sourceLineNo">107</span> // Select most recent backup id<a name="line.107"></a>
116 <span class="sourceLineNo">108</span> String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds);<a name="line.108"></a>
117 <span class="sourceLineNo">109</span><a name="line.109"></a>
118 <span class="sourceLineNo">110</span> TableName[] tableNames = getTableNamesInBackupImages(backupIds);<a name="line.110"></a>
119 <span class="sourceLineNo">111</span><a name="line.111"></a>
120 <span class="sourceLineNo">112</span> BackupInfo bInfo = table.readBackupInfo(backupIds[0]);<a name="line.112"></a>
121 <span class="sourceLineNo">113</span> String backupRoot = bInfo.getBackupRootDir();<a name="line.113"></a>
122 <span class="sourceLineNo">114</span><a name="line.114"></a>
123 <span class="sourceLineNo">115</span> for (int i = 0; i &lt; tableNames.length; i++) {<a name="line.115"></a>
124 <span class="sourceLineNo">116</span> LOG.info("Merge backup images for " + tableNames[i]);<a name="line.116"></a>
125 <span class="sourceLineNo">117</span><a name="line.117"></a>
126 <span class="sourceLineNo">118</span> // Find input directories for table<a name="line.118"></a>
127 <span class="sourceLineNo">119</span> Path[] dirPaths = findInputDirectories(fs, backupRoot, tableNames[i], backupIds);<a name="line.119"></a>
128 <span class="sourceLineNo">120</span> String dirs = StringUtils.join(dirPaths, ",");<a name="line.120"></a>
129 <span class="sourceLineNo">121</span><a name="line.121"></a>
130 <span class="sourceLineNo">122</span> Path bulkOutputPath =<a name="line.122"></a>
131 <span class="sourceLineNo">123</span> BackupUtils.getBulkOutputDir(BackupUtils.getFileNameCompatibleString(tableNames[i]),<a name="line.123"></a>
132 <span class="sourceLineNo">124</span> getConf(), false);<a name="line.124"></a>
133 <span class="sourceLineNo">125</span> // Delete content if exists<a name="line.125"></a>
134 <span class="sourceLineNo">126</span> if (fs.exists(bulkOutputPath)) {<a name="line.126"></a>
135 <span class="sourceLineNo">127</span> if (!fs.delete(bulkOutputPath, true)) {<a name="line.127"></a>
136 <span class="sourceLineNo">128</span> LOG.warn("Can not delete: " + bulkOutputPath);<a name="line.128"></a>
137 <span class="sourceLineNo">129</span> }<a name="line.129"></a>
138 <span class="sourceLineNo">130</span> }<a name="line.130"></a>
139 <span class="sourceLineNo">131</span> Configuration conf = getConf();<a name="line.131"></a>
140 <span class="sourceLineNo">132</span> conf.set(bulkOutputConfKey, bulkOutputPath.toString());<a name="line.132"></a>
141 <span class="sourceLineNo">133</span> String[] playerArgs = { dirs, tableNames[i].getNameAsString() };<a name="line.133"></a>
142 <span class="sourceLineNo">134</span><a name="line.134"></a>
143 <span class="sourceLineNo">135</span> player.setConf(getConf());<a name="line.135"></a>
144 <span class="sourceLineNo">136</span> int result = player.run(playerArgs);<a name="line.136"></a>
145 <span class="sourceLineNo">137</span> if (!succeeded(result)) {<a name="line.137"></a>
146 <span class="sourceLineNo">138</span> throw new IOException("Can not merge backup images for " + dirs<a name="line.138"></a>
147 <span class="sourceLineNo">139</span> + " (check Hadoop/MR and HBase logs). Player return code =" + result);<a name="line.139"></a>
148 <span class="sourceLineNo">140</span> }<a name="line.140"></a>
149 <span class="sourceLineNo">141</span> // Add to processed table list<a name="line.141"></a>
150 <span class="sourceLineNo">142</span> processedTableList.add(new Pair&lt;&gt;(tableNames[i], bulkOutputPath));<a name="line.142"></a>
151 <span class="sourceLineNo">143</span> LOG.debug("Merge Job finished:" + result);<a name="line.143"></a>
152 <span class="sourceLineNo">144</span> }<a name="line.144"></a>
153 <span class="sourceLineNo">145</span> List&lt;TableName&gt; tableList = toTableNameList(processedTableList);<a name="line.145"></a>
154 <span class="sourceLineNo">146</span> table.updateProcessedTablesForMerge(tableList);<a name="line.146"></a>
155 <span class="sourceLineNo">147</span> finishedTables = true;<a name="line.147"></a>
156 <span class="sourceLineNo">148</span><a name="line.148"></a>
157 <span class="sourceLineNo">149</span> // PHASE 2 (modification of a backup file system)<a name="line.149"></a>
158 <span class="sourceLineNo">150</span> // Move existing mergedBackupId data into tmp directory<a name="line.150"></a>
159 <span class="sourceLineNo">151</span> // we will need it later in case of a failure<a name="line.151"></a>
160 <span class="sourceLineNo">152</span> Path tmpBackupDir = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot,<a name="line.152"></a>
161 <span class="sourceLineNo">153</span> mergedBackupId);<a name="line.153"></a>
162 <span class="sourceLineNo">154</span> Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, mergedBackupId);<a name="line.154"></a>
163 <span class="sourceLineNo">155</span><a name="line.155"></a>
164 <span class="sourceLineNo">156</span> if (!fs.rename(backupDirPath, tmpBackupDir)) {<a name="line.156"></a>
165 <span class="sourceLineNo">157</span> throw new IOException("Failed to rename "+ backupDirPath +" to "+tmpBackupDir);<a name="line.157"></a>
166 <span class="sourceLineNo">158</span> } else {<a name="line.158"></a>
167 <span class="sourceLineNo">159</span> LOG.debug("Renamed "+ backupDirPath +" to "+ tmpBackupDir);<a name="line.159"></a>
168 <span class="sourceLineNo">160</span> }<a name="line.160"></a>
169 <span class="sourceLineNo">161</span> // Move new data into backup dest<a name="line.161"></a>
170 <span class="sourceLineNo">162</span> for (Pair&lt;TableName, Path&gt; tn : processedTableList) {<a name="line.162"></a>
171 <span class="sourceLineNo">163</span> moveData(fs, backupRoot, tn.getSecond(), tn.getFirst(), mergedBackupId);<a name="line.163"></a>
172 <span class="sourceLineNo">164</span> }<a name="line.164"></a>
173 <span class="sourceLineNo">165</span> // Update backup manifest<a name="line.165"></a>
174 <span class="sourceLineNo">166</span> List&lt;String&gt; backupsToDelete = getBackupIdsToDelete(backupIds, mergedBackupId);<a name="line.166"></a>
175 <span class="sourceLineNo">167</span> updateBackupManifest(tmpBackupDir.getParent().toString(), mergedBackupId, backupsToDelete);<a name="line.167"></a>
176 <span class="sourceLineNo">168</span> // Copy meta files back from tmp to backup dir<a name="line.168"></a>
177 <span class="sourceLineNo">169</span> copyMetaData(fs, tmpBackupDir, backupDirPath);<a name="line.169"></a>
178 <span class="sourceLineNo">170</span> // Delete tmp dir (Rename back during repair)<a name="line.170"></a>
179 <span class="sourceLineNo">171</span> if (!fs.delete(tmpBackupDir, true)) {<a name="line.171"></a>
180 <span class="sourceLineNo">172</span> // WARN and ignore<a name="line.172"></a>
181 <span class="sourceLineNo">173</span> LOG.warn("Could not delete tmp dir: "+ tmpBackupDir);<a name="line.173"></a>
182 <span class="sourceLineNo">174</span> }<a name="line.174"></a>
183 <span class="sourceLineNo">175</span> // Delete old data<a name="line.175"></a>
184 <span class="sourceLineNo">176</span> deleteBackupImages(backupsToDelete, conn, fs, backupRoot);<a name="line.176"></a>
185 <span class="sourceLineNo">177</span> // Finish merge session<a name="line.177"></a>
186 <span class="sourceLineNo">178</span> table.finishMergeOperation();<a name="line.178"></a>
187 <span class="sourceLineNo">179</span> // Release lock<a name="line.179"></a>
188 <span class="sourceLineNo">180</span> table.finishBackupExclusiveOperation();<a name="line.180"></a>
189 <span class="sourceLineNo">181</span> } catch (RuntimeException e) {<a name="line.181"></a>
190 <span class="sourceLineNo">182</span><a name="line.182"></a>
191 <span class="sourceLineNo">183</span> throw e;<a name="line.183"></a>
192 <span class="sourceLineNo">184</span> } catch (Exception e) {<a name="line.184"></a>
193 <span class="sourceLineNo">185</span> LOG.error(e.toString(), e);<a name="line.185"></a>
194 <span class="sourceLineNo">186</span> if (!finishedTables) {<a name="line.186"></a>
195 <span class="sourceLineNo">187</span> // cleanup bulk directories and finish merge<a name="line.187"></a>
196 <span class="sourceLineNo">188</span> // merge MUST be repeated (no need for repair)<a name="line.188"></a>
197 <span class="sourceLineNo">189</span> cleanupBulkLoadDirs(fs, toPathList(processedTableList));<a name="line.189"></a>
198 <span class="sourceLineNo">190</span> table.finishMergeOperation();<a name="line.190"></a>
199 <span class="sourceLineNo">191</span> table.finishBackupExclusiveOperation();<a name="line.191"></a>
200 <span class="sourceLineNo">192</span> throw new IOException("Backup merge operation failed, you should try it again", e);<a name="line.192"></a>
201 <span class="sourceLineNo">193</span> } else {<a name="line.193"></a>
202 <span class="sourceLineNo">194</span> // backup repair must be run<a name="line.194"></a>
203 <span class="sourceLineNo">195</span> throw new IOException(<a name="line.195"></a>
204 <span class="sourceLineNo">196</span> "Backup merge operation failed, run backup repair tool to restore system's integrity",<a name="line.196"></a>
205 <span class="sourceLineNo">197</span> e);<a name="line.197"></a>
206 <span class="sourceLineNo">198</span> }<a name="line.198"></a>
207 <span class="sourceLineNo">199</span> } finally {<a name="line.199"></a>
208 <span class="sourceLineNo">200</span> table.close();<a name="line.200"></a>
209 <span class="sourceLineNo">201</span> conn.close();<a name="line.201"></a>
210 <span class="sourceLineNo">202</span> }<a name="line.202"></a>
211 <span class="sourceLineNo">203</span> }<a name="line.203"></a>
212 <span class="sourceLineNo">204</span><a name="line.204"></a>
213 <span class="sourceLineNo">205</span> /**<a name="line.205"></a>
214 <span class="sourceLineNo">206</span> * Copy meta data to of a backup session<a name="line.206"></a>
215 <span class="sourceLineNo">207</span> * @param fs file system<a name="line.207"></a>
216 <span class="sourceLineNo">208</span> * @param tmpBackupDir temp backup directory, where meta is locaed<a name="line.208"></a>
217 <span class="sourceLineNo">209</span> * @param backupDirPath new path for backup<a name="line.209"></a>
218 <span class="sourceLineNo">210</span> * @throws IOException exception<a name="line.210"></a>
219 <span class="sourceLineNo">211</span> */<a name="line.211"></a>
220 <span class="sourceLineNo">212</span> protected void copyMetaData(FileSystem fs, Path tmpBackupDir, Path backupDirPath)<a name="line.212"></a>
221 <span class="sourceLineNo">213</span> throws IOException {<a name="line.213"></a>
222 <span class="sourceLineNo">214</span> RemoteIterator&lt;LocatedFileStatus&gt; it = fs.listFiles(tmpBackupDir, true);<a name="line.214"></a>
223 <span class="sourceLineNo">215</span> List&lt;Path&gt; toKeep = new ArrayList&lt;Path&gt;();<a name="line.215"></a>
224 <span class="sourceLineNo">216</span> while (it.hasNext()) {<a name="line.216"></a>
225 <span class="sourceLineNo">217</span> Path p = it.next().getPath();<a name="line.217"></a>
226 <span class="sourceLineNo">218</span> if (fs.isDirectory(p)) {<a name="line.218"></a>
227 <span class="sourceLineNo">219</span> continue;<a name="line.219"></a>
228 <span class="sourceLineNo">220</span> }<a name="line.220"></a>
229 <span class="sourceLineNo">221</span> // Keep meta<a name="line.221"></a>
230 <span class="sourceLineNo">222</span> String fileName = p.toString();<a name="line.222"></a>
231 <span class="sourceLineNo">223</span> if (fileName.indexOf(FSTableDescriptors.TABLEINFO_DIR) &gt; 0<a name="line.223"></a>
232 <span class="sourceLineNo">224</span> || fileName.indexOf(HRegionFileSystem.REGION_INFO_FILE) &gt; 0) {<a name="line.224"></a>
233 <span class="sourceLineNo">225</span> toKeep.add(p);<a name="line.225"></a>
234 <span class="sourceLineNo">226</span> }<a name="line.226"></a>
235 <span class="sourceLineNo">227</span> }<a name="line.227"></a>
236 <span class="sourceLineNo">228</span> // Copy meta to destination<a name="line.228"></a>
237 <span class="sourceLineNo">229</span> for (Path p : toKeep) {<a name="line.229"></a>
238 <span class="sourceLineNo">230</span> Path newPath = convertToDest(p, backupDirPath);<a name="line.230"></a>
239 <span class="sourceLineNo">231</span> copyFile(fs, p, newPath);<a name="line.231"></a>
240 <span class="sourceLineNo">232</span> }<a name="line.232"></a>
241 <span class="sourceLineNo">233</span> }<a name="line.233"></a>
242 <span class="sourceLineNo">234</span><a name="line.234"></a>
243 <span class="sourceLineNo">235</span> /**<a name="line.235"></a>
244 <span class="sourceLineNo">236</span> * Copy file in DFS from p to newPath<a name="line.236"></a>
245 <span class="sourceLineNo">237</span> * @param fs file system<a name="line.237"></a>
246 <span class="sourceLineNo">238</span> * @param p old path<a name="line.238"></a>
247 <span class="sourceLineNo">239</span> * @param newPath new path<a name="line.239"></a>
248 <span class="sourceLineNo">240</span> * @throws IOException exception<a name="line.240"></a>
249 <span class="sourceLineNo">241</span> */<a name="line.241"></a>
250 <span class="sourceLineNo">242</span> protected void copyFile(FileSystem fs, Path p, Path newPath) throws IOException {<a name="line.242"></a>
251 <span class="sourceLineNo">243</span> File f = File.createTempFile("data", "meta");<a name="line.243"></a>
252 <span class="sourceLineNo">244</span> Path localPath = new Path(f.getAbsolutePath());<a name="line.244"></a>
253 <span class="sourceLineNo">245</span> fs.copyToLocalFile(p, localPath);<a name="line.245"></a>
254 <span class="sourceLineNo">246</span> fs.copyFromLocalFile(localPath, newPath);<a name="line.246"></a>
255 <span class="sourceLineNo">247</span> boolean exists = fs.exists(newPath);<a name="line.247"></a>
256 <span class="sourceLineNo">248</span> if (!exists) {<a name="line.248"></a>
257 <span class="sourceLineNo">249</span> throw new IOException("Failed to copy meta file to: "+ newPath);<a name="line.249"></a>
258 <span class="sourceLineNo">250</span> }<a name="line.250"></a>
259 <span class="sourceLineNo">251</span> }<a name="line.251"></a>
260 <span class="sourceLineNo">252</span><a name="line.252"></a>
261 <span class="sourceLineNo">253</span>/**<a name="line.253"></a>
262 <span class="sourceLineNo">254</span> * Converts path before copying<a name="line.254"></a>
263 <span class="sourceLineNo">255</span> * @param p path<a name="line.255"></a>
264 <span class="sourceLineNo">256</span> * @param backupDirPath backup root<a name="line.256"></a>
265 <span class="sourceLineNo">257</span> * @return converted path<a name="line.257"></a>
266 <span class="sourceLineNo">258</span> */<a name="line.258"></a>
267 <span class="sourceLineNo">259</span> protected Path convertToDest(Path p, Path backupDirPath) {<a name="line.259"></a>
268 <span class="sourceLineNo">260</span> String backupId = backupDirPath.getName();<a name="line.260"></a>
269 <span class="sourceLineNo">261</span> Stack&lt;String&gt; stack = new Stack&lt;String&gt;();<a name="line.261"></a>
270 <span class="sourceLineNo">262</span> String name = null;<a name="line.262"></a>
271 <span class="sourceLineNo">263</span> while (true) {<a name="line.263"></a>
272 <span class="sourceLineNo">264</span> name = p.getName();<a name="line.264"></a>
273 <span class="sourceLineNo">265</span> if (!name.equals(backupId)) {<a name="line.265"></a>
274 <span class="sourceLineNo">266</span> stack.push(name);<a name="line.266"></a>
275 <span class="sourceLineNo">267</span> p = p.getParent();<a name="line.267"></a>
276 <span class="sourceLineNo">268</span> } else {<a name="line.268"></a>
277 <span class="sourceLineNo">269</span> break;<a name="line.269"></a>
278 <span class="sourceLineNo">270</span> }<a name="line.270"></a>
279 <span class="sourceLineNo">271</span> }<a name="line.271"></a>
280 <span class="sourceLineNo">272</span> Path newPath = new Path(backupDirPath.toString());<a name="line.272"></a>
281 <span class="sourceLineNo">273</span> while (!stack.isEmpty()) {<a name="line.273"></a>
282 <span class="sourceLineNo">274</span> newPath = new Path(newPath, stack.pop());<a name="line.274"></a>
283 <span class="sourceLineNo">275</span> }<a name="line.275"></a>
284 <span class="sourceLineNo">276</span> return newPath;<a name="line.276"></a>
285 <span class="sourceLineNo">277</span> }<a name="line.277"></a>
286 <span class="sourceLineNo">278</span><a name="line.278"></a>
287 <span class="sourceLineNo">279</span> protected List&lt;Path&gt; toPathList(List&lt;Pair&lt;TableName, Path&gt;&gt; processedTableList) {<a name="line.279"></a>
288 <span class="sourceLineNo">280</span> ArrayList&lt;Path&gt; list = new ArrayList&lt;&gt;();<a name="line.280"></a>
289 <span class="sourceLineNo">281</span> for (Pair&lt;TableName, Path&gt; p : processedTableList) {<a name="line.281"></a>
290 <span class="sourceLineNo">282</span> list.add(p.getSecond());<a name="line.282"></a>
291 <span class="sourceLineNo">283</span> }<a name="line.283"></a>
292 <span class="sourceLineNo">284</span> return list;<a name="line.284"></a>
293 <span class="sourceLineNo">285</span> }<a name="line.285"></a>
294 <span class="sourceLineNo">286</span><a name="line.286"></a>
295 <span class="sourceLineNo">287</span> protected List&lt;TableName&gt; toTableNameList(List&lt;Pair&lt;TableName, Path&gt;&gt; processedTableList) {<a name="line.287"></a>
296 <span class="sourceLineNo">288</span> ArrayList&lt;TableName&gt; list = new ArrayList&lt;&gt;();<a name="line.288"></a>
297 <span class="sourceLineNo">289</span> for (Pair&lt;TableName, Path&gt; p : processedTableList) {<a name="line.289"></a>
298 <span class="sourceLineNo">290</span> list.add(p.getFirst());<a name="line.290"></a>
299 <span class="sourceLineNo">291</span> }<a name="line.291"></a>
300 <span class="sourceLineNo">292</span> return list;<a name="line.292"></a>
301 <span class="sourceLineNo">293</span> }<a name="line.293"></a>
302 <span class="sourceLineNo">294</span><a name="line.294"></a>
303 <span class="sourceLineNo">295</span> protected void cleanupBulkLoadDirs(FileSystem fs, List&lt;Path&gt; pathList) throws IOException {<a name="line.295"></a>
304 <span class="sourceLineNo">296</span> for (Path path : pathList) {<a name="line.296"></a>
305 <span class="sourceLineNo">297</span> if (!fs.delete(path, true)) {<a name="line.297"></a>
306 <span class="sourceLineNo">298</span> LOG.warn("Can't delete " + path);<a name="line.298"></a>
307 <span class="sourceLineNo">299</span> }<a name="line.299"></a>
308 <span class="sourceLineNo">300</span> }<a name="line.300"></a>
309 <span class="sourceLineNo">301</span> }<a name="line.301"></a>
310 <span class="sourceLineNo">302</span><a name="line.302"></a>
311 <span class="sourceLineNo">303</span> protected void updateBackupManifest(String backupRoot, String mergedBackupId,<a name="line.303"></a>
312 <span class="sourceLineNo">304</span> List&lt;String&gt; backupsToDelete) throws IllegalArgumentException, IOException {<a name="line.304"></a>
313 <span class="sourceLineNo">305</span> BackupManifest manifest =<a name="line.305"></a>
314 <span class="sourceLineNo">306</span> HBackupFileSystem.getManifest(conf, new Path(backupRoot), mergedBackupId);<a name="line.306"></a>
315 <span class="sourceLineNo">307</span> manifest.getBackupImage().removeAncestors(backupsToDelete);<a name="line.307"></a>
316 <span class="sourceLineNo">308</span> // save back<a name="line.308"></a>
317 <span class="sourceLineNo">309</span> manifest.store(conf);<a name="line.309"></a>
318 <span class="sourceLineNo">310</span> }<a name="line.310"></a>
319 <span class="sourceLineNo">311</span><a name="line.311"></a>
320 <span class="sourceLineNo">312</span> protected void deleteBackupImages(List&lt;String&gt; backupIds, Connection conn, FileSystem fs,<a name="line.312"></a>
321 <span class="sourceLineNo">313</span> String backupRoot) throws IOException {<a name="line.313"></a>
322 <span class="sourceLineNo">314</span> // Delete from backup system table<a name="line.314"></a>
323 <span class="sourceLineNo">315</span> try (BackupSystemTable table = new BackupSystemTable(conn)) {<a name="line.315"></a>
324 <span class="sourceLineNo">316</span> for (String backupId : backupIds) {<a name="line.316"></a>
325 <span class="sourceLineNo">317</span> table.deleteBackupInfo(backupId);<a name="line.317"></a>
326 <span class="sourceLineNo">318</span> }<a name="line.318"></a>
327 <span class="sourceLineNo">319</span> }<a name="line.319"></a>
328 <span class="sourceLineNo">320</span><a name="line.320"></a>
329 <span class="sourceLineNo">321</span> // Delete from file system<a name="line.321"></a>
330 <span class="sourceLineNo">322</span> for (String backupId : backupIds) {<a name="line.322"></a>
331 <span class="sourceLineNo">323</span> Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, backupId);<a name="line.323"></a>
332 <span class="sourceLineNo">324</span><a name="line.324"></a>
333 <span class="sourceLineNo">325</span> if (!fs.delete(backupDirPath, true)) {<a name="line.325"></a>
334 <span class="sourceLineNo">326</span> LOG.warn("Could not delete " + backupDirPath);<a name="line.326"></a>
335 <span class="sourceLineNo">327</span> }<a name="line.327"></a>
336 <span class="sourceLineNo">328</span> }<a name="line.328"></a>
337 <span class="sourceLineNo">329</span> }<a name="line.329"></a>
338 <span class="sourceLineNo">330</span><a name="line.330"></a>
339 <span class="sourceLineNo">331</span> protected List&lt;String&gt; getBackupIdsToDelete(String[] backupIds, String mergedBackupId) {<a name="line.331"></a>
340 <span class="sourceLineNo">332</span> List&lt;String&gt; list = new ArrayList&lt;&gt;();<a name="line.332"></a>
341 <span class="sourceLineNo">333</span> for (String id : backupIds) {<a name="line.333"></a>
342 <span class="sourceLineNo">334</span> if (id.equals(mergedBackupId)) {<a name="line.334"></a>
343 <span class="sourceLineNo">335</span> continue;<a name="line.335"></a>
344 <span class="sourceLineNo">336</span> }<a name="line.336"></a>
345 <span class="sourceLineNo">337</span> list.add(id);<a name="line.337"></a>
346 <span class="sourceLineNo">338</span> }<a name="line.338"></a>
347 <span class="sourceLineNo">339</span> return list;<a name="line.339"></a>
348 <span class="sourceLineNo">340</span> }<a name="line.340"></a>
349 <span class="sourceLineNo">341</span><a name="line.341"></a>
350 <span class="sourceLineNo">342</span> protected void moveData(FileSystem fs, String backupRoot, Path bulkOutputPath,<a name="line.342"></a>
351 <span class="sourceLineNo">343</span> TableName tableName, String mergedBackupId) throws IllegalArgumentException, IOException {<a name="line.343"></a>
352 <span class="sourceLineNo">344</span> Path dest =<a name="line.344"></a>
353 <span class="sourceLineNo">345</span> new Path(HBackupFileSystem.getTableBackupDir(backupRoot, mergedBackupId, tableName));<a name="line.345"></a>
354 <span class="sourceLineNo">346</span><a name="line.346"></a>
355 <span class="sourceLineNo">347</span> FileStatus[] fsts = fs.listStatus(bulkOutputPath);<a name="line.347"></a>
356 <span class="sourceLineNo">348</span> for (FileStatus fst : fsts) {<a name="line.348"></a>
357 <span class="sourceLineNo">349</span> if (fst.isDirectory()) {<a name="line.349"></a>
358 <span class="sourceLineNo">350</span> String family = fst.getPath().getName();<a name="line.350"></a>
359 <span class="sourceLineNo">351</span> Path newDst = new Path(dest, family);<a name="line.351"></a>
360 <span class="sourceLineNo">352</span> if (fs.exists(newDst)) {<a name="line.352"></a>
361 <span class="sourceLineNo">353</span> if (!fs.delete(newDst, true)) {<a name="line.353"></a>
362 <span class="sourceLineNo">354</span> throw new IOException("failed to delete :"+ newDst);<a name="line.354"></a>
363 <span class="sourceLineNo">355</span> }<a name="line.355"></a>
364 <span class="sourceLineNo">356</span> } else {<a name="line.356"></a>
365 <span class="sourceLineNo">357</span> fs.mkdirs(dest);<a name="line.357"></a>
366 <span class="sourceLineNo">358</span> }<a name="line.358"></a>
367 <span class="sourceLineNo">359</span> boolean result = fs.rename(fst.getPath(), dest);<a name="line.359"></a>
368 <span class="sourceLineNo">360</span> LOG.debug("MoveData from "+ fst.getPath() +" to "+ dest+" result="+ result);<a name="line.360"></a>
369 <span class="sourceLineNo">361</span> }<a name="line.361"></a>
370 <span class="sourceLineNo">362</span> }<a name="line.362"></a>
371 <span class="sourceLineNo">363</span> }<a name="line.363"></a>
372 <span class="sourceLineNo">364</span><a name="line.364"></a>
373 <span class="sourceLineNo">365</span> protected TableName[] getTableNamesInBackupImages(String[] backupIds) throws IOException {<a name="line.365"></a>
374 <span class="sourceLineNo">366</span> Set&lt;TableName&gt; allSet = new HashSet&lt;&gt;();<a name="line.366"></a>
375 <span class="sourceLineNo">367</span><a name="line.367"></a>
376 <span class="sourceLineNo">368</span> try (Connection conn = ConnectionFactory.createConnection(conf);<a name="line.368"></a>
377 <span class="sourceLineNo">369</span> BackupSystemTable table = new BackupSystemTable(conn)) {<a name="line.369"></a>
378 <span class="sourceLineNo">370</span> for (String backupId : backupIds) {<a name="line.370"></a>
379 <span class="sourceLineNo">371</span> BackupInfo bInfo = table.readBackupInfo(backupId);<a name="line.371"></a>
380 <span class="sourceLineNo">372</span><a name="line.372"></a>
381 <span class="sourceLineNo">373</span> allSet.addAll(bInfo.getTableNames());<a name="line.373"></a>
382 <span class="sourceLineNo">374</span> }<a name="line.374"></a>
383 <span class="sourceLineNo">375</span> }<a name="line.375"></a>
384 <span class="sourceLineNo">376</span><a name="line.376"></a>
385 <span class="sourceLineNo">377</span> TableName[] ret = new TableName[allSet.size()];<a name="line.377"></a>
386 <span class="sourceLineNo">378</span> return allSet.toArray(ret);<a name="line.378"></a>
387 <span class="sourceLineNo">379</span> }<a name="line.379"></a>
388 <span class="sourceLineNo">380</span><a name="line.380"></a>
389 <span class="sourceLineNo">381</span> protected Path[] findInputDirectories(FileSystem fs, String backupRoot, TableName tableName,<a name="line.381"></a>
390 <span class="sourceLineNo">382</span> String[] backupIds) throws IOException {<a name="line.382"></a>
391 <span class="sourceLineNo">383</span> List&lt;Path&gt; dirs = new ArrayList&lt;&gt;();<a name="line.383"></a>
392 <span class="sourceLineNo">384</span><a name="line.384"></a>
393 <span class="sourceLineNo">385</span> for (String backupId : backupIds) {<a name="line.385"></a>
394 <span class="sourceLineNo">386</span> Path fileBackupDirPath =<a name="line.386"></a>
395 <span class="sourceLineNo">387</span> new Path(HBackupFileSystem.getTableBackupDir(backupRoot, backupId, tableName));<a name="line.387"></a>
396 <span class="sourceLineNo">388</span> if (fs.exists(fileBackupDirPath)) {<a name="line.388"></a>
397 <span class="sourceLineNo">389</span> dirs.add(fileBackupDirPath);<a name="line.389"></a>
398 <span class="sourceLineNo">390</span> } else {<a name="line.390"></a>
399 <span class="sourceLineNo">391</span> if (LOG.isDebugEnabled()) {<a name="line.391"></a>
400 <span class="sourceLineNo">392</span> LOG.debug("File: " + fileBackupDirPath + " does not exist.");<a name="line.392"></a>
401 <span class="sourceLineNo">393</span> }<a name="line.393"></a>
402 <span class="sourceLineNo">394</span> }<a name="line.394"></a>
403 <span class="sourceLineNo">395</span> }<a name="line.395"></a>
404 <span class="sourceLineNo">396</span> Path[] ret = new Path[dirs.size()];<a name="line.396"></a>
405 <span class="sourceLineNo">397</span> return dirs.toArray(ret);<a name="line.397"></a>
406 <span class="sourceLineNo">398</span> }<a name="line.398"></a>
407 <span class="sourceLineNo">399</span>}<a name="line.399"></a>
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468 </pre>
469 </div>
470 </body>
471 </html>