devapidocs/src-html/org/apache/hadoop/hbase/backup/util/BackupUtils.html

   1 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
   2 <html lang="en">
   3 <head>
   4 <title>Source code</title>
   5 <link rel="stylesheet" type="text/css" href="../../../../../../../stylesheet.css" title="Style">
   6 </head>
   7 <body>
   8 <div class="sourceContainer">
   9 <pre><span class="sourceLineNo">001</span>/**<a name="line.1"></a>
  10 <span class="sourceLineNo">002</span> * Licensed to the Apache Software Foundation (ASF) under one<a name="line.2"></a>
  11 <span class="sourceLineNo">003</span> * or more contributor license agreements.  See the NOTICE file<a name="line.3"></a>
  12 <span class="sourceLineNo">004</span> * distributed with this work for additional information<a name="line.4"></a>
  13 <span class="sourceLineNo">005</span> * regarding copyright ownership.  The ASF licenses this file<a name="line.5"></a>
  14 <span class="sourceLineNo">006</span> * to you under the Apache License, Version 2.0 (the<a name="line.6"></a>
  15 <span class="sourceLineNo">007</span> * "License"); you may not use this file except in compliance<a name="line.7"></a>
  16 <span class="sourceLineNo">008</span> * with the License.  You may obtain a copy of the License at<a name="line.8"></a>
  17 <span class="sourceLineNo">009</span> *<a name="line.9"></a>
  18 <span class="sourceLineNo">010</span> *     http://www.apache.org/licenses/LICENSE-2.0<a name="line.10"></a>
  19 <span class="sourceLineNo">011</span> *<a name="line.11"></a>
  20 <span class="sourceLineNo">012</span> * Unless required by applicable law or agreed to in writing, software<a name="line.12"></a>
  21 <span class="sourceLineNo">013</span> * distributed under the License is distributed on an "AS IS" BASIS,<a name="line.13"></a>
  22 <span class="sourceLineNo">014</span> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.<a name="line.14"></a>
  23 <span class="sourceLineNo">015</span> * See the License for the specific language governing permissions and<a name="line.15"></a>
  24 <span class="sourceLineNo">016</span> * limitations under the License.<a name="line.16"></a>
  25 <span class="sourceLineNo">017</span> */<a name="line.17"></a>
  26 <span class="sourceLineNo">018</span><a name="line.18"></a>
  27 <span class="sourceLineNo">019</span>package org.apache.hadoop.hbase.backup.util;<a name="line.19"></a>
  28 <span class="sourceLineNo">020</span><a name="line.20"></a>
  29 <span class="sourceLineNo">021</span>import java.io.FileNotFoundException;<a name="line.21"></a>
  30 <span class="sourceLineNo">022</span>import java.io.IOException;<a name="line.22"></a>
  31 <span class="sourceLineNo">023</span>import java.net.URLDecoder;<a name="line.23"></a>
  32 <span class="sourceLineNo">024</span>import java.util.ArrayList;<a name="line.24"></a>
  33 <span class="sourceLineNo">025</span>import java.util.Collections;<a name="line.25"></a>
  34 <span class="sourceLineNo">026</span>import java.util.Comparator;<a name="line.26"></a>
  35 <span class="sourceLineNo">027</span>import java.util.HashMap;<a name="line.27"></a>
  36 <span class="sourceLineNo">028</span>import java.util.Iterator;<a name="line.28"></a>
  37 <span class="sourceLineNo">029</span>import java.util.List;<a name="line.29"></a>
  38 <span class="sourceLineNo">030</span>import java.util.Map.Entry;<a name="line.30"></a>
  39 <span class="sourceLineNo">031</span>import java.util.TreeMap;<a name="line.31"></a>
  40 <span class="sourceLineNo">032</span>import java.util.TreeSet;<a name="line.32"></a>
  41 <span class="sourceLineNo">033</span><a name="line.33"></a>
  42 <span class="sourceLineNo">034</span>import org.apache.hadoop.conf.Configuration;<a name="line.34"></a>
  43 <span class="sourceLineNo">035</span>import org.apache.hadoop.fs.FSDataOutputStream;<a name="line.35"></a>
  44 <span class="sourceLineNo">036</span>import org.apache.hadoop.fs.FileStatus;<a name="line.36"></a>
  45 <span class="sourceLineNo">037</span>import org.apache.hadoop.fs.FileSystem;<a name="line.37"></a>
  46 <span class="sourceLineNo">038</span>import org.apache.hadoop.fs.LocatedFileStatus;<a name="line.38"></a>
  47 <span class="sourceLineNo">039</span>import org.apache.hadoop.fs.Path;<a name="line.39"></a>
  48 <span class="sourceLineNo">040</span>import org.apache.hadoop.fs.PathFilter;<a name="line.40"></a>
  49 <span class="sourceLineNo">041</span>import org.apache.hadoop.fs.RemoteIterator;<a name="line.41"></a>
  50 <span class="sourceLineNo">042</span>import org.apache.hadoop.fs.permission.FsPermission;<a name="line.42"></a>
  51 <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.HConstants;<a name="line.43"></a>
  52 <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.MetaTableAccessor;<a name="line.44"></a>
  53 <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.ServerName;<a name="line.45"></a>
  54 <span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.TableName;<a name="line.46"></a>
  55 <span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.backup.BackupInfo;<a name="line.47"></a>
  56 <span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.backup.BackupRestoreConstants;<a name="line.48"></a>
  57 <span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.backup.HBackupFileSystem;<a name="line.49"></a>
  58 <span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.backup.RestoreRequest;<a name="line.50"></a>
  59 <span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.backup.impl.BackupManifest;<a name="line.51"></a>
  60 <span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;<a name="line.52"></a>
  61 <span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.client.Connection;<a name="line.53"></a>
  62 <span class="sourceLineNo">054</span>import org.apache.hadoop.hbase.client.RegionInfo;<a name="line.54"></a>
  63 <span class="sourceLineNo">055</span>import org.apache.hadoop.hbase.client.TableDescriptor;<a name="line.55"></a>
  64 <span class="sourceLineNo">056</span>import org.apache.hadoop.hbase.regionserver.HRegion;<a name="line.56"></a>
  65 <span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles;<a name="line.57"></a>
  66 <span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.58"></a>
  67 <span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.util.FSTableDescriptors;<a name="line.59"></a>
  68 <span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.60"></a>
  69 <span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;<a name="line.61"></a>
  70 <span class="sourceLineNo">062</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.62"></a>
  71 <span class="sourceLineNo">063</span>import org.slf4j.Logger;<a name="line.63"></a>
  72 <span class="sourceLineNo">064</span>import org.slf4j.LoggerFactory;<a name="line.64"></a>
  73 <span class="sourceLineNo">065</span><a name="line.65"></a>
  74 <span class="sourceLineNo">066</span>/**<a name="line.66"></a>
  75 <span class="sourceLineNo">067</span> * A collection for methods used by multiple classes to backup HBase tables.<a name="line.67"></a>
  76 <span class="sourceLineNo">068</span> */<a name="line.68"></a>
  77 <span class="sourceLineNo">069</span>@InterfaceAudience.Private<a name="line.69"></a>
  78 <span class="sourceLineNo">070</span>public final class BackupUtils {<a name="line.70"></a>
  79 <span class="sourceLineNo">071</span>  protected static final Logger LOG = LoggerFactory.getLogger(BackupUtils.class);<a name="line.71"></a>
  80 <span class="sourceLineNo">072</span>  public static final String LOGNAME_SEPARATOR = ".";<a name="line.72"></a>
  81 <span class="sourceLineNo">073</span>  public static final int MILLISEC_IN_HOUR = 3600000;<a name="line.73"></a>
  82 <span class="sourceLineNo">074</span><a name="line.74"></a>
  83 <span class="sourceLineNo">075</span>  private BackupUtils() {<a name="line.75"></a>
  84 <span class="sourceLineNo">076</span>    throw new AssertionError("Instantiating utility class...");<a name="line.76"></a>
  85 <span class="sourceLineNo">077</span>  }<a name="line.77"></a>
  86 <span class="sourceLineNo">078</span><a name="line.78"></a>
  87 <span class="sourceLineNo">079</span>  /**<a name="line.79"></a>
  88 <span class="sourceLineNo">080</span>   * Loop through the RS log timestamp map for the tables, for each RS, find the min timestamp value<a name="line.80"></a>
  89 <span class="sourceLineNo">081</span>   * for the RS among the tables.<a name="line.81"></a>
  90 <span class="sourceLineNo">082</span>   * @param rsLogTimestampMap timestamp map<a name="line.82"></a>
  91 <span class="sourceLineNo">083</span>   * @return the min timestamp of each RS<a name="line.83"></a>
  92 <span class="sourceLineNo">084</span>   */<a name="line.84"></a>
  93 <span class="sourceLineNo">085</span>  public static HashMap&lt;String, Long&gt; getRSLogTimestampMins(<a name="line.85"></a>
  94 <span class="sourceLineNo">086</span>      HashMap&lt;TableName, HashMap&lt;String, Long&gt;&gt; rsLogTimestampMap) {<a name="line.86"></a>
  95 <span class="sourceLineNo">087</span>    if (rsLogTimestampMap == null || rsLogTimestampMap.isEmpty()) {<a name="line.87"></a>
  96 <span class="sourceLineNo">088</span>      return null;<a name="line.88"></a>
  97 <span class="sourceLineNo">089</span>    }<a name="line.89"></a>
  98 <span class="sourceLineNo">090</span><a name="line.90"></a>
  99 <span class="sourceLineNo">091</span>    HashMap&lt;String, Long&gt; rsLogTimestampMins = new HashMap&lt;&gt;();<a name="line.91"></a>
 100 <span class="sourceLineNo">092</span>    HashMap&lt;String, HashMap&lt;TableName, Long&gt;&gt; rsLogTimestampMapByRS = new HashMap&lt;&gt;();<a name="line.92"></a>
 101 <span class="sourceLineNo">093</span><a name="line.93"></a>
 102 <span class="sourceLineNo">094</span>    for (Entry&lt;TableName, HashMap&lt;String, Long&gt;&gt; tableEntry : rsLogTimestampMap.entrySet()) {<a name="line.94"></a>
 103 <span class="sourceLineNo">095</span>      TableName table = tableEntry.getKey();<a name="line.95"></a>
 104 <span class="sourceLineNo">096</span>      HashMap&lt;String, Long&gt; rsLogTimestamp = tableEntry.getValue();<a name="line.96"></a>
 105 <span class="sourceLineNo">097</span>      for (Entry&lt;String, Long&gt; rsEntry : rsLogTimestamp.entrySet()) {<a name="line.97"></a>
 106 <span class="sourceLineNo">098</span>        String rs = rsEntry.getKey();<a name="line.98"></a>
 107 <span class="sourceLineNo">099</span>        Long ts = rsEntry.getValue();<a name="line.99"></a>
 108 <span class="sourceLineNo">100</span>        if (!rsLogTimestampMapByRS.containsKey(rs)) {<a name="line.100"></a>
 109 <span class="sourceLineNo">101</span>          rsLogTimestampMapByRS.put(rs, new HashMap&lt;&gt;());<a name="line.101"></a>
 110 <span class="sourceLineNo">102</span>          rsLogTimestampMapByRS.get(rs).put(table, ts);<a name="line.102"></a>
 111 <span class="sourceLineNo">103</span>        } else {<a name="line.103"></a>
 112 <span class="sourceLineNo">104</span>          rsLogTimestampMapByRS.get(rs).put(table, ts);<a name="line.104"></a>
 113 <span class="sourceLineNo">105</span>        }<a name="line.105"></a>
 114 <span class="sourceLineNo">106</span>      }<a name="line.106"></a>
 115 <span class="sourceLineNo">107</span>    }<a name="line.107"></a>
 116 <span class="sourceLineNo">108</span><a name="line.108"></a>
 117 <span class="sourceLineNo">109</span>    for (Entry&lt;String, HashMap&lt;TableName, Long&gt;&gt; entry : rsLogTimestampMapByRS.entrySet()) {<a name="line.109"></a>
 118 <span class="sourceLineNo">110</span>      String rs = entry.getKey();<a name="line.110"></a>
 119 <span class="sourceLineNo">111</span>      rsLogTimestampMins.put(rs, BackupUtils.getMinValue(entry.getValue()));<a name="line.111"></a>
 120 <span class="sourceLineNo">112</span>    }<a name="line.112"></a>
 121 <span class="sourceLineNo">113</span><a name="line.113"></a>
 122 <span class="sourceLineNo">114</span>    return rsLogTimestampMins;<a name="line.114"></a>
 123 <span class="sourceLineNo">115</span>  }<a name="line.115"></a>
 124 <span class="sourceLineNo">116</span><a name="line.116"></a>
 125 <span class="sourceLineNo">117</span>  /**<a name="line.117"></a>
 126 <span class="sourceLineNo">118</span>   * copy out Table RegionInfo into incremental backup image need to consider move this logic into<a name="line.118"></a>
 127 <span class="sourceLineNo">119</span>   * HBackupFileSystem<a name="line.119"></a>
 128 <span class="sourceLineNo">120</span>   * @param conn connection<a name="line.120"></a>
 129 <span class="sourceLineNo">121</span>   * @param backupInfo backup info<a name="line.121"></a>
 130 <span class="sourceLineNo">122</span>   * @param conf configuration<a name="line.122"></a>
 131 <span class="sourceLineNo">123</span>   * @throws IOException exception<a name="line.123"></a>
 132 <span class="sourceLineNo">124</span>   */<a name="line.124"></a>
 133 <span class="sourceLineNo">125</span>  public static void copyTableRegionInfo(Connection conn, BackupInfo backupInfo, Configuration conf)<a name="line.125"></a>
 134 <span class="sourceLineNo">126</span>          throws IOException {<a name="line.126"></a>
 135 <span class="sourceLineNo">127</span>    Path rootDir = FSUtils.getRootDir(conf);<a name="line.127"></a>
 136 <span class="sourceLineNo">128</span>    FileSystem fs = rootDir.getFileSystem(conf);<a name="line.128"></a>
 137 <span class="sourceLineNo">129</span><a name="line.129"></a>
 138 <span class="sourceLineNo">130</span>    // for each table in the table set, copy out the table info and region<a name="line.130"></a>
 139 <span class="sourceLineNo">131</span>    // info files in the correct directory structure<a name="line.131"></a>
 140 <span class="sourceLineNo">132</span>    for (TableName table : backupInfo.getTables()) {<a name="line.132"></a>
 141 <span class="sourceLineNo">133</span>      if (!MetaTableAccessor.tableExists(conn, table)) {<a name="line.133"></a>
 142 <span class="sourceLineNo">134</span>        LOG.warn("Table " + table + " does not exists, skipping it.");<a name="line.134"></a>
 143 <span class="sourceLineNo">135</span>        continue;<a name="line.135"></a>
 144 <span class="sourceLineNo">136</span>      }<a name="line.136"></a>
 145 <span class="sourceLineNo">137</span>      TableDescriptor orig = FSTableDescriptors.getTableDescriptorFromFs(fs, rootDir, table);<a name="line.137"></a>
 146 <span class="sourceLineNo">138</span><a name="line.138"></a>
 147 <span class="sourceLineNo">139</span>      // write a copy of descriptor to the target directory<a name="line.139"></a>
 148 <span class="sourceLineNo">140</span>      Path target = new Path(backupInfo.getTableBackupDir(table));<a name="line.140"></a>
 149 <span class="sourceLineNo">141</span>      FileSystem targetFs = target.getFileSystem(conf);<a name="line.141"></a>
 150 <span class="sourceLineNo">142</span>      FSTableDescriptors descriptors =<a name="line.142"></a>
 151 <span class="sourceLineNo">143</span>          new FSTableDescriptors(conf, targetFs, FSUtils.getRootDir(conf));<a name="line.143"></a>
 152 <span class="sourceLineNo">144</span>      descriptors.createTableDescriptorForTableDirectory(target, orig, false);<a name="line.144"></a>
 153 <span class="sourceLineNo">145</span>      LOG.debug("Attempting to copy table info for:" + table + " target: " + target<a name="line.145"></a>
 154 <span class="sourceLineNo">146</span>          + " descriptor: " + orig);<a name="line.146"></a>
 155 <span class="sourceLineNo">147</span>      LOG.debug("Finished copying tableinfo.");<a name="line.147"></a>
 156 <span class="sourceLineNo">148</span>      List&lt;RegionInfo&gt; regions = MetaTableAccessor.getTableRegions(conn, table);<a name="line.148"></a>
 157 <span class="sourceLineNo">149</span>      // For each region, write the region info to disk<a name="line.149"></a>
 158 <span class="sourceLineNo">150</span>      LOG.debug("Starting to write region info for table " + table);<a name="line.150"></a>
 159 <span class="sourceLineNo">151</span>      for (RegionInfo regionInfo : regions) {<a name="line.151"></a>
 160 <span class="sourceLineNo">152</span>        Path regionDir =<a name="line.152"></a>
 161 <span class="sourceLineNo">153</span>            HRegion.getRegionDir(new Path(backupInfo.getTableBackupDir(table)), regionInfo);<a name="line.153"></a>
 162 <span class="sourceLineNo">154</span>        regionDir = new Path(backupInfo.getTableBackupDir(table), regionDir.getName());<a name="line.154"></a>
 163 <span class="sourceLineNo">155</span>        writeRegioninfoOnFilesystem(conf, targetFs, regionDir, regionInfo);<a name="line.155"></a>
 164 <span class="sourceLineNo">156</span>      }<a name="line.156"></a>
 165 <span class="sourceLineNo">157</span>      LOG.debug("Finished writing region info for table " + table);<a name="line.157"></a>
 166 <span class="sourceLineNo">158</span>    }<a name="line.158"></a>
 167 <span class="sourceLineNo">159</span>  }<a name="line.159"></a>
 168 <span class="sourceLineNo">160</span><a name="line.160"></a>
 169 <span class="sourceLineNo">161</span>  /**<a name="line.161"></a>
 170 <span class="sourceLineNo">162</span>   * Write the .regioninfo file on-disk.<a name="line.162"></a>
 171 <span class="sourceLineNo">163</span>   */<a name="line.163"></a>
 172 <span class="sourceLineNo">164</span>  public static void writeRegioninfoOnFilesystem(final Configuration conf, final FileSystem fs,<a name="line.164"></a>
 173 <span class="sourceLineNo">165</span>      final Path regionInfoDir, RegionInfo regionInfo) throws IOException {<a name="line.165"></a>
 174 <span class="sourceLineNo">166</span>    final byte[] content = RegionInfo.toDelimitedByteArray(regionInfo);<a name="line.166"></a>
 175 <span class="sourceLineNo">167</span>    Path regionInfoFile = new Path(regionInfoDir, "." + HConstants.REGIONINFO_QUALIFIER_STR);<a name="line.167"></a>
 176 <span class="sourceLineNo">168</span>    // First check to get the permissions<a name="line.168"></a>
 177 <span class="sourceLineNo">169</span>    FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);<a name="line.169"></a>
 178 <span class="sourceLineNo">170</span>    // Write the RegionInfo file content<a name="line.170"></a>
 179 <span class="sourceLineNo">171</span>    FSDataOutputStream out = FSUtils.create(conf, fs, regionInfoFile, perms, null);<a name="line.171"></a>
 180 <span class="sourceLineNo">172</span>    try {<a name="line.172"></a>
 181 <span class="sourceLineNo">173</span>      out.write(content);<a name="line.173"></a>
 182 <span class="sourceLineNo">174</span>    } finally {<a name="line.174"></a>
 183 <span class="sourceLineNo">175</span>      out.close();<a name="line.175"></a>
 184 <span class="sourceLineNo">176</span>    }<a name="line.176"></a>
 185 <span class="sourceLineNo">177</span>  }<a name="line.177"></a>
 186 <span class="sourceLineNo">178</span><a name="line.178"></a>
 187 <span class="sourceLineNo">179</span>  /**<a name="line.179"></a>
 188 <span class="sourceLineNo">180</span>   * Parses hostname:port from WAL file path<a name="line.180"></a>
 189 <span class="sourceLineNo">181</span>   * @param p path to WAL file<a name="line.181"></a>
 190 <span class="sourceLineNo">182</span>   * @return hostname:port<a name="line.182"></a>
 191 <span class="sourceLineNo">183</span>   */<a name="line.183"></a>
 192 <span class="sourceLineNo">184</span>  public static String parseHostNameFromLogFile(Path p) {<a name="line.184"></a>
 193 <span class="sourceLineNo">185</span>    try {<a name="line.185"></a>
 194 <span class="sourceLineNo">186</span>      if (AbstractFSWALProvider.isArchivedLogFile(p)) {<a name="line.186"></a>
 195 <span class="sourceLineNo">187</span>        return BackupUtils.parseHostFromOldLog(p);<a name="line.187"></a>
 196 <span class="sourceLineNo">188</span>      } else {<a name="line.188"></a>
 197 <span class="sourceLineNo">189</span>        ServerName sname = AbstractFSWALProvider.getServerNameFromWALDirectoryName(p);<a name="line.189"></a>
 198 <span class="sourceLineNo">190</span>        if (sname != null) {<a name="line.190"></a>
 199 <span class="sourceLineNo">191</span>          return sname.getAddress().toString();<a name="line.191"></a>
 200 <span class="sourceLineNo">192</span>        } else {<a name="line.192"></a>
 201 <span class="sourceLineNo">193</span>          LOG.error("Skip log file (can't parse): " + p);<a name="line.193"></a>
 202 <span class="sourceLineNo">194</span>          return null;<a name="line.194"></a>
 203 <span class="sourceLineNo">195</span>        }<a name="line.195"></a>
 204 <span class="sourceLineNo">196</span>      }<a name="line.196"></a>
 205 <span class="sourceLineNo">197</span>    } catch (Exception e) {<a name="line.197"></a>
 206 <span class="sourceLineNo">198</span>      LOG.error("Skip log file (can't parse): " + p, e);<a name="line.198"></a>
 207 <span class="sourceLineNo">199</span>      return null;<a name="line.199"></a>
 208 <span class="sourceLineNo">200</span>    }<a name="line.200"></a>
 209 <span class="sourceLineNo">201</span>  }<a name="line.201"></a>
 210 <span class="sourceLineNo">202</span><a name="line.202"></a>
 211 <span class="sourceLineNo">203</span>  /**<a name="line.203"></a>
 212 <span class="sourceLineNo">204</span>   * Returns WAL file name<a name="line.204"></a>
 213 <span class="sourceLineNo">205</span>   * @param walFileName WAL file name<a name="line.205"></a>
 214 <span class="sourceLineNo">206</span>   * @return WAL file name<a name="line.206"></a>
 215 <span class="sourceLineNo">207</span>   */<a name="line.207"></a>
 216 <span class="sourceLineNo">208</span>  public static String getUniqueWALFileNamePart(String walFileName) {<a name="line.208"></a>
 217 <span class="sourceLineNo">209</span>    return getUniqueWALFileNamePart(new Path(walFileName));<a name="line.209"></a>
 218 <span class="sourceLineNo">210</span>  }<a name="line.210"></a>
 219 <span class="sourceLineNo">211</span><a name="line.211"></a>
 220 <span class="sourceLineNo">212</span>  /**<a name="line.212"></a>
 221 <span class="sourceLineNo">213</span>   * Returns WAL file name<a name="line.213"></a>
 222 <span class="sourceLineNo">214</span>   * @param p WAL file path<a name="line.214"></a>
 223 <span class="sourceLineNo">215</span>   * @return WAL file name<a name="line.215"></a>
 224 <span class="sourceLineNo">216</span>   */<a name="line.216"></a>
 225 <span class="sourceLineNo">217</span>  public static String getUniqueWALFileNamePart(Path p) {<a name="line.217"></a>
 226 <span class="sourceLineNo">218</span>    return p.getName();<a name="line.218"></a>
 227 <span class="sourceLineNo">219</span>  }<a name="line.219"></a>
 228 <span class="sourceLineNo">220</span><a name="line.220"></a>
 229 <span class="sourceLineNo">221</span>  /**<a name="line.221"></a>
 230 <span class="sourceLineNo">222</span>   * Get the total length of files under the given directory recursively.<a name="line.222"></a>
 231 <span class="sourceLineNo">223</span>   * @param fs The hadoop file system<a name="line.223"></a>
 232 <span class="sourceLineNo">224</span>   * @param dir The target directory<a name="line.224"></a>
 233 <span class="sourceLineNo">225</span>   * @return the total length of files<a name="line.225"></a>
 234 <span class="sourceLineNo">226</span>   * @throws IOException exception<a name="line.226"></a>
 235 <span class="sourceLineNo">227</span>   */<a name="line.227"></a>
 236 <span class="sourceLineNo">228</span>  public static long getFilesLength(FileSystem fs, Path dir) throws IOException {<a name="line.228"></a>
 237 <span class="sourceLineNo">229</span>    long totalLength = 0;<a name="line.229"></a>
 238 <span class="sourceLineNo">230</span>    FileStatus[] files = FSUtils.listStatus(fs, dir);<a name="line.230"></a>
 239 <span class="sourceLineNo">231</span>    if (files != null) {<a name="line.231"></a>
 240 <span class="sourceLineNo">232</span>      for (FileStatus fileStatus : files) {<a name="line.232"></a>
 241 <span class="sourceLineNo">233</span>        if (fileStatus.isDirectory()) {<a name="line.233"></a>
 242 <span class="sourceLineNo">234</span>          totalLength += getFilesLength(fs, fileStatus.getPath());<a name="line.234"></a>
 243 <span class="sourceLineNo">235</span>        } else {<a name="line.235"></a>
 244 <span class="sourceLineNo">236</span>          totalLength += fileStatus.getLen();<a name="line.236"></a>
 245 <span class="sourceLineNo">237</span>        }<a name="line.237"></a>
 246 <span class="sourceLineNo">238</span>      }<a name="line.238"></a>
 247 <span class="sourceLineNo">239</span>    }<a name="line.239"></a>
 248 <span class="sourceLineNo">240</span>    return totalLength;<a name="line.240"></a>
 249 <span class="sourceLineNo">241</span>  }<a name="line.241"></a>
 250 <span class="sourceLineNo">242</span><a name="line.242"></a>
 251 <span class="sourceLineNo">243</span>  /**<a name="line.243"></a>
 252 <span class="sourceLineNo">244</span>   * Get list of all old WAL files (WALs and archive)<a name="line.244"></a>
 253 <span class="sourceLineNo">245</span>   * @param c configuration<a name="line.245"></a>
 254 <span class="sourceLineNo">246</span>   * @param hostTimestampMap {host,timestamp} map<a name="line.246"></a>
 255 <span class="sourceLineNo">247</span>   * @return list of WAL files<a name="line.247"></a>
 256 <span class="sourceLineNo">248</span>   * @throws IOException exception<a name="line.248"></a>
 257 <span class="sourceLineNo">249</span>   */<a name="line.249"></a>
 258 <span class="sourceLineNo">250</span>  public static List&lt;String&gt; getWALFilesOlderThan(final Configuration c,<a name="line.250"></a>
 259 <span class="sourceLineNo">251</span>      final HashMap&lt;String, Long&gt; hostTimestampMap) throws IOException {<a name="line.251"></a>
 260 <span class="sourceLineNo">252</span>    Path rootDir = FSUtils.getRootDir(c);<a name="line.252"></a>
 261 <span class="sourceLineNo">253</span>    Path logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);<a name="line.253"></a>
 262 <span class="sourceLineNo">254</span>    Path oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.254"></a>
 263 <span class="sourceLineNo">255</span>    List&lt;String&gt; logFiles = new ArrayList&lt;&gt;();<a name="line.255"></a>
 264 <span class="sourceLineNo">256</span><a name="line.256"></a>
 265 <span class="sourceLineNo">257</span>    PathFilter filter = p -&gt; {<a name="line.257"></a>
 266 <span class="sourceLineNo">258</span>      try {<a name="line.258"></a>
 267 <span class="sourceLineNo">259</span>        if (AbstractFSWALProvider.isMetaFile(p)) {<a name="line.259"></a>
 268 <span class="sourceLineNo">260</span>          return false;<a name="line.260"></a>
 269 <span class="sourceLineNo">261</span>        }<a name="line.261"></a>
 270 <span class="sourceLineNo">262</span>        String host = parseHostNameFromLogFile(p);<a name="line.262"></a>
 271 <span class="sourceLineNo">263</span>        if (host == null) {<a name="line.263"></a>
 272 <span class="sourceLineNo">264</span>          return false;<a name="line.264"></a>
 273 <span class="sourceLineNo">265</span>        }<a name="line.265"></a>
 274 <span class="sourceLineNo">266</span>        Long oldTimestamp = hostTimestampMap.get(host);<a name="line.266"></a>
 275 <span class="sourceLineNo">267</span>        Long currentLogTS = BackupUtils.getCreationTime(p);<a name="line.267"></a>
 276 <span class="sourceLineNo">268</span>        return currentLogTS &lt;= oldTimestamp;<a name="line.268"></a>
 277 <span class="sourceLineNo">269</span>      } catch (Exception e) {<a name="line.269"></a>
 278 <span class="sourceLineNo">270</span>        LOG.warn("Can not parse" + p, e);<a name="line.270"></a>
 279 <span class="sourceLineNo">271</span>        return false;<a name="line.271"></a>
 280 <span class="sourceLineNo">272</span>      }<a name="line.272"></a>
 281 <span class="sourceLineNo">273</span>    };<a name="line.273"></a>
 282 <span class="sourceLineNo">274</span>    FileSystem fs = FileSystem.get(c);<a name="line.274"></a>
 283 <span class="sourceLineNo">275</span>    logFiles = BackupUtils.getFiles(fs, logDir, logFiles, filter);<a name="line.275"></a>
 284 <span class="sourceLineNo">276</span>    logFiles = BackupUtils.getFiles(fs, oldLogDir, logFiles, filter);<a name="line.276"></a>
 285 <span class="sourceLineNo">277</span>    return logFiles;<a name="line.277"></a>
 286 <span class="sourceLineNo">278</span>  }<a name="line.278"></a>
 287 <span class="sourceLineNo">279</span><a name="line.279"></a>
 288 <span class="sourceLineNo">280</span>  public static TableName[] parseTableNames(String tables) {<a name="line.280"></a>
 289 <span class="sourceLineNo">281</span>    if (tables == null) {<a name="line.281"></a>
 290 <span class="sourceLineNo">282</span>      return null;<a name="line.282"></a>
 291 <span class="sourceLineNo">283</span>    }<a name="line.283"></a>
 292 <span class="sourceLineNo">284</span>    String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);<a name="line.284"></a>
 293 <span class="sourceLineNo">285</span><a name="line.285"></a>
 294 <span class="sourceLineNo">286</span>    TableName[] ret = new TableName[tableArray.length];<a name="line.286"></a>
 295 <span class="sourceLineNo">287</span>    for (int i = 0; i &lt; tableArray.length; i++) {<a name="line.287"></a>
 296 <span class="sourceLineNo">288</span>      ret[i] = TableName.valueOf(tableArray[i]);<a name="line.288"></a>
 297 <span class="sourceLineNo">289</span>    }<a name="line.289"></a>
 298 <span class="sourceLineNo">290</span>    return ret;<a name="line.290"></a>
 299 <span class="sourceLineNo">291</span>  }<a name="line.291"></a>
 300 <span class="sourceLineNo">292</span><a name="line.292"></a>
 301 <span class="sourceLineNo">293</span>  /**<a name="line.293"></a>
 302 <span class="sourceLineNo">294</span>   * Check whether the backup path exist<a name="line.294"></a>
 303 <span class="sourceLineNo">295</span>   * @param backupStr backup<a name="line.295"></a>
 304 <span class="sourceLineNo">296</span>   * @param conf configuration<a name="line.296"></a>
 305 <span class="sourceLineNo">297</span>   * @return Yes if path exists<a name="line.297"></a>
 306 <span class="sourceLineNo">298</span>   * @throws IOException exception<a name="line.298"></a>
 307 <span class="sourceLineNo">299</span>   */<a name="line.299"></a>
 308 <span class="sourceLineNo">300</span>  public static boolean checkPathExist(String backupStr, Configuration conf) throws IOException {<a name="line.300"></a>
 309 <span class="sourceLineNo">301</span>    boolean isExist = false;<a name="line.301"></a>
 310 <span class="sourceLineNo">302</span>    Path backupPath = new Path(backupStr);<a name="line.302"></a>
 311 <span class="sourceLineNo">303</span>    FileSystem fileSys = backupPath.getFileSystem(conf);<a name="line.303"></a>
 312 <span class="sourceLineNo">304</span>    String targetFsScheme = fileSys.getUri().getScheme();<a name="line.304"></a>
 313 <span class="sourceLineNo">305</span>    if (LOG.isTraceEnabled()) {<a name="line.305"></a>
 314 <span class="sourceLineNo">306</span>      LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme);<a name="line.306"></a>
 315 <span class="sourceLineNo">307</span>    }<a name="line.307"></a>
 316 <span class="sourceLineNo">308</span>    if (fileSys.exists(backupPath)) {<a name="line.308"></a>
 317 <span class="sourceLineNo">309</span>      isExist = true;<a name="line.309"></a>
 318 <span class="sourceLineNo">310</span>    }<a name="line.310"></a>
 319 <span class="sourceLineNo">311</span>    return isExist;<a name="line.311"></a>
 320 <span class="sourceLineNo">312</span>  }<a name="line.312"></a>
 321 <span class="sourceLineNo">313</span><a name="line.313"></a>
 322 <span class="sourceLineNo">314</span>  /**<a name="line.314"></a>
 323 <span class="sourceLineNo">315</span>   * Check target path first, confirm it doesn't exist before backup<a name="line.315"></a>
 324 <span class="sourceLineNo">316</span>   * @param backupRootPath backup destination path<a name="line.316"></a>
 325 <span class="sourceLineNo">317</span>   * @param conf configuration<a name="line.317"></a>
 326 <span class="sourceLineNo">318</span>   * @throws IOException exception<a name="line.318"></a>
 327 <span class="sourceLineNo">319</span>   */<a name="line.319"></a>
 328 <span class="sourceLineNo">320</span>  public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException {<a name="line.320"></a>
 329 <span class="sourceLineNo">321</span>    boolean targetExists;<a name="line.321"></a>
 330 <span class="sourceLineNo">322</span>    try {<a name="line.322"></a>
 331 <span class="sourceLineNo">323</span>      targetExists = checkPathExist(backupRootPath, conf);<a name="line.323"></a>
 332 <span class="sourceLineNo">324</span>    } catch (IOException e) {<a name="line.324"></a>
 333 <span class="sourceLineNo">325</span>      String expMsg = e.getMessage();<a name="line.325"></a>
 334 <span class="sourceLineNo">326</span>      String newMsg = null;<a name="line.326"></a>
 335 <span class="sourceLineNo">327</span>      if (expMsg.contains("No FileSystem for scheme")) {<a name="line.327"></a>
 336 <span class="sourceLineNo">328</span>        newMsg =<a name="line.328"></a>
 337 <span class="sourceLineNo">329</span>            "Unsupported filesystem scheme found in the backup target url. Error Message: "<a name="line.329"></a>
 338 <span class="sourceLineNo">330</span>                + newMsg;<a name="line.330"></a>
 339 <span class="sourceLineNo">331</span>        LOG.error(newMsg);<a name="line.331"></a>
 340 <span class="sourceLineNo">332</span>        throw new IOException(newMsg);<a name="line.332"></a>
 341 <span class="sourceLineNo">333</span>      } else {<a name="line.333"></a>
 342 <span class="sourceLineNo">334</span>        throw e;<a name="line.334"></a>
 343 <span class="sourceLineNo">335</span>      }<a name="line.335"></a>
 344 <span class="sourceLineNo">336</span>    }<a name="line.336"></a>
 345 <span class="sourceLineNo">337</span><a name="line.337"></a>
 346 <span class="sourceLineNo">338</span>    if (targetExists) {<a name="line.338"></a>
 347 <span class="sourceLineNo">339</span>      LOG.info("Using existing backup root dir: " + backupRootPath);<a name="line.339"></a>
 348 <span class="sourceLineNo">340</span>    } else {<a name="line.340"></a>
 349 <span class="sourceLineNo">341</span>      LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");<a name="line.341"></a>
 350 <span class="sourceLineNo">342</span>    }<a name="line.342"></a>
 351 <span class="sourceLineNo">343</span>  }<a name="line.343"></a>
 352 <span class="sourceLineNo">344</span><a name="line.344"></a>
 353 <span class="sourceLineNo">345</span>  /**<a name="line.345"></a>
 354 <span class="sourceLineNo">346</span>   * Get the min value for all the Values a map.<a name="line.346"></a>
 355 <span class="sourceLineNo">347</span>   * @param map map<a name="line.347"></a>
 356 <span class="sourceLineNo">348</span>   * @return the min value<a name="line.348"></a>
 357 <span class="sourceLineNo">349</span>   */<a name="line.349"></a>
 358 <span class="sourceLineNo">350</span>  public static &lt;T&gt; Long getMinValue(HashMap&lt;T, Long&gt; map) {<a name="line.350"></a>
 359 <span class="sourceLineNo">351</span>    Long minTimestamp = null;<a name="line.351"></a>
 360 <span class="sourceLineNo">352</span>    if (map != null) {<a name="line.352"></a>
 361 <span class="sourceLineNo">353</span>      ArrayList&lt;Long&gt; timestampList = new ArrayList&lt;&gt;(map.values());<a name="line.353"></a>
 362 <span class="sourceLineNo">354</span>      Collections.sort(timestampList);<a name="line.354"></a>
 363 <span class="sourceLineNo">355</span>      // The min among all the RS log timestamps will be kept in backup system table table.<a name="line.355"></a>
 364 <span class="sourceLineNo">356</span>      minTimestamp = timestampList.get(0);<a name="line.356"></a>
 365 <span class="sourceLineNo">357</span>    }<a name="line.357"></a>
 366 <span class="sourceLineNo">358</span>    return minTimestamp;<a name="line.358"></a>
 367 <span class="sourceLineNo">359</span>  }<a name="line.359"></a>
 368 <span class="sourceLineNo">360</span><a name="line.360"></a>
 369 <span class="sourceLineNo">361</span>  /**<a name="line.361"></a>
 370 <span class="sourceLineNo">362</span>   * Parses host name:port from archived WAL path<a name="line.362"></a>
 371 <span class="sourceLineNo">363</span>   * @param p path<a name="line.363"></a>
 372 <span class="sourceLineNo">364</span>   * @return host name<a name="line.364"></a>
 373 <span class="sourceLineNo">365</span>   */<a name="line.365"></a>
 374 <span class="sourceLineNo">366</span>  public static String parseHostFromOldLog(Path p) {<a name="line.366"></a>
 375 <span class="sourceLineNo">367</span>    try {<a name="line.367"></a>
 376 <span class="sourceLineNo">368</span>      String n = p.getName();<a name="line.368"></a>
 377 <span class="sourceLineNo">369</span>      int idx = n.lastIndexOf(LOGNAME_SEPARATOR);<a name="line.369"></a>
 378 <span class="sourceLineNo">370</span>      String s = URLDecoder.decode(n.substring(0, idx), "UTF8");<a name="line.370"></a>
 379 <span class="sourceLineNo">371</span>      return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);<a name="line.371"></a>
 380 <span class="sourceLineNo">372</span>    } catch (Exception e) {<a name="line.372"></a>
 381 <span class="sourceLineNo">373</span>      LOG.warn("Skip log file (can't parse): " + p);<a name="line.373"></a>
 382 <span class="sourceLineNo">374</span>      return null;<a name="line.374"></a>
 383 <span class="sourceLineNo">375</span>    }<a name="line.375"></a>
 384 <span class="sourceLineNo">376</span>  }<a name="line.376"></a>
 385 <span class="sourceLineNo">377</span><a name="line.377"></a>
 386 <span class="sourceLineNo">378</span>  /**<a name="line.378"></a>
 387 <span class="sourceLineNo">379</span>   * Given the log file, parse the timestamp from the file name. The timestamp is the last number.<a name="line.379"></a>
 388 <span class="sourceLineNo">380</span>   * @param p a path to the log file<a name="line.380"></a>
 389 <span class="sourceLineNo">381</span>   * @return the timestamp<a name="line.381"></a>
 390 <span class="sourceLineNo">382</span>   * @throws IOException exception<a name="line.382"></a>
 391 <span class="sourceLineNo">383</span>   */<a name="line.383"></a>
 392 <span class="sourceLineNo">384</span>  public static Long getCreationTime(Path p) throws IOException {<a name="line.384"></a>
 393 <span class="sourceLineNo">385</span>    int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);<a name="line.385"></a>
 394 <span class="sourceLineNo">386</span>    if (idx &lt; 0) {<a name="line.386"></a>
 395 <span class="sourceLineNo">387</span>      throw new IOException("Cannot parse timestamp from path " + p);<a name="line.387"></a>
 396 <span class="sourceLineNo">388</span>    }<a name="line.388"></a>
 397 <span class="sourceLineNo">389</span>    String ts = p.getName().substring(idx + 1);<a name="line.389"></a>
 398 <span class="sourceLineNo">390</span>    return Long.parseLong(ts);<a name="line.390"></a>
 399 <span class="sourceLineNo">391</span>  }<a name="line.391"></a>
 400 <span class="sourceLineNo">392</span><a name="line.392"></a>
 401 <span class="sourceLineNo">393</span>  public static List&lt;String&gt; getFiles(FileSystem fs, Path rootDir, List&lt;String&gt; files,<a name="line.393"></a>
 402 <span class="sourceLineNo">394</span>      PathFilter filter) throws IOException {<a name="line.394"></a>
 403 <span class="sourceLineNo">395</span>    RemoteIterator&lt;LocatedFileStatus&gt; it = fs.listFiles(rootDir, true);<a name="line.395"></a>
 404 <span class="sourceLineNo">396</span><a name="line.396"></a>
 405 <span class="sourceLineNo">397</span>    while (it.hasNext()) {<a name="line.397"></a>
 406 <span class="sourceLineNo">398</span>      LocatedFileStatus lfs = it.next();<a name="line.398"></a>
 407 <span class="sourceLineNo">399</span>      if (lfs.isDirectory()) {<a name="line.399"></a>
 408 <span class="sourceLineNo">400</span>        continue;<a name="line.400"></a>
 409 <span class="sourceLineNo">401</span>      }<a name="line.401"></a>
 410 <span class="sourceLineNo">402</span>      // apply filter<a name="line.402"></a>
 411 <span class="sourceLineNo">403</span>      if (filter.accept(lfs.getPath())) {<a name="line.403"></a>
 412 <span class="sourceLineNo">404</span>        files.add(lfs.getPath().toString());<a name="line.404"></a>
 413 <span class="sourceLineNo">405</span>      }<a name="line.405"></a>
 414 <span class="sourceLineNo">406</span>    }<a name="line.406"></a>
 415 <span class="sourceLineNo">407</span>    return files;<a name="line.407"></a>
 416 <span class="sourceLineNo">408</span>  }<a name="line.408"></a>
 417 <span class="sourceLineNo">409</span><a name="line.409"></a>
 418 <span class="sourceLineNo">410</span>  public static void cleanupBackupData(BackupInfo context, Configuration conf) throws IOException {<a name="line.410"></a>
 419 <span class="sourceLineNo">411</span>    cleanupHLogDir(context, conf);<a name="line.411"></a>
 420 <span class="sourceLineNo">412</span>    cleanupTargetDir(context, conf);<a name="line.412"></a>
 421 <span class="sourceLineNo">413</span>  }<a name="line.413"></a>
 422 <span class="sourceLineNo">414</span><a name="line.414"></a>
 423 <span class="sourceLineNo">415</span>  /**<a name="line.415"></a>
 424 <span class="sourceLineNo">416</span>   * Clean up directories which are generated when DistCp copying hlogs<a name="line.416"></a>
 425 <span class="sourceLineNo">417</span>   * @param backupInfo backup info<a name="line.417"></a>
 426 <span class="sourceLineNo">418</span>   * @param conf configuration<a name="line.418"></a>
 427 <span class="sourceLineNo">419</span>   * @throws IOException exception<a name="line.419"></a>
 428 <span class="sourceLineNo">420</span>   */<a name="line.420"></a>
 429 <span class="sourceLineNo">421</span>  private static void cleanupHLogDir(BackupInfo backupInfo, Configuration conf) throws IOException {<a name="line.421"></a>
 430 <span class="sourceLineNo">422</span>    String logDir = backupInfo.getHLogTargetDir();<a name="line.422"></a>
 431 <span class="sourceLineNo">423</span>    if (logDir == null) {<a name="line.423"></a>
 432 <span class="sourceLineNo">424</span>      LOG.warn("No log directory specified for " + backupInfo.getBackupId());<a name="line.424"></a>
 433 <span class="sourceLineNo">425</span>      return;<a name="line.425"></a>
 434 <span class="sourceLineNo">426</span>    }<a name="line.426"></a>
 435 <span class="sourceLineNo">427</span><a name="line.427"></a>
 436 <span class="sourceLineNo">428</span>    Path rootPath = new Path(logDir).getParent();<a name="line.428"></a>
 437 <span class="sourceLineNo">429</span>    FileSystem fs = FileSystem.get(rootPath.toUri(), conf);<a name="line.429"></a>
 438 <span class="sourceLineNo">430</span>    FileStatus[] files = listStatus(fs, rootPath, null);<a name="line.430"></a>
 439 <span class="sourceLineNo">431</span>    if (files == null) {<a name="line.431"></a>
 440 <span class="sourceLineNo">432</span>      return;<a name="line.432"></a>
 441 <span class="sourceLineNo">433</span>    }<a name="line.433"></a>
 442 <span class="sourceLineNo">434</span>    for (FileStatus file : files) {<a name="line.434"></a>
 443 <span class="sourceLineNo">435</span>      LOG.debug("Delete log files: " + file.getPath().getName());<a name="line.435"></a>
 444 <span class="sourceLineNo">436</span>      fs.delete(file.getPath(), true);<a name="line.436"></a>
 445 <span class="sourceLineNo">437</span>    }<a name="line.437"></a>
 446 <span class="sourceLineNo">438</span>  }<a name="line.438"></a>
 447 <span class="sourceLineNo">439</span><a name="line.439"></a>
 448 <span class="sourceLineNo">440</span>  private static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) {<a name="line.440"></a>
 449 <span class="sourceLineNo">441</span>    try {<a name="line.441"></a>
 450 <span class="sourceLineNo">442</span>      // clean up the data at target directory<a name="line.442"></a>
 451 <span class="sourceLineNo">443</span>      LOG.debug("Trying to cleanup up target dir : " + backupInfo.getBackupId());<a name="line.443"></a>
 452 <span class="sourceLineNo">444</span>      String targetDir = backupInfo.getBackupRootDir();<a name="line.444"></a>
 453 <span class="sourceLineNo">445</span>      if (targetDir == null) {<a name="line.445"></a>
 454 <span class="sourceLineNo">446</span>        LOG.warn("No target directory specified for " + backupInfo.getBackupId());<a name="line.446"></a>
 455 <span class="sourceLineNo">447</span>        return;<a name="line.447"></a>
 456 <span class="sourceLineNo">448</span>      }<a name="line.448"></a>
 457 <span class="sourceLineNo">449</span><a name="line.449"></a>
 458 <span class="sourceLineNo">450</span>      FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf);<a name="line.450"></a>
 459 <span class="sourceLineNo">451</span><a name="line.451"></a>
 460 <span class="sourceLineNo">452</span>      for (TableName table : backupInfo.getTables()) {<a name="line.452"></a>
 461 <span class="sourceLineNo">453</span>        Path targetDirPath =<a name="line.453"></a>
 462 <span class="sourceLineNo">454</span>            new Path(getTableBackupDir(backupInfo.getBackupRootDir(), backupInfo.getBackupId(),<a name="line.454"></a>
 463 <span class="sourceLineNo">455</span>              table));<a name="line.455"></a>
 464 <span class="sourceLineNo">456</span>        if (outputFs.delete(targetDirPath, true)) {<a name="line.456"></a>
 465 <span class="sourceLineNo">457</span>          LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");<a name="line.457"></a>
 466 <span class="sourceLineNo">458</span>        } else {<a name="line.458"></a>
 467 <span class="sourceLineNo">459</span>          LOG.info("No data has been found in " + targetDirPath.toString() + ".");<a name="line.459"></a>
 468 <span class="sourceLineNo">460</span>        }<a name="line.460"></a>
 469 <span class="sourceLineNo">461</span><a name="line.461"></a>
 470 <span class="sourceLineNo">462</span>        Path tableDir = targetDirPath.getParent();<a name="line.462"></a>
 471 <span class="sourceLineNo">463</span>        FileStatus[] backups = listStatus(outputFs, tableDir, null);<a name="line.463"></a>
 472 <span class="sourceLineNo">464</span>        if (backups == null || backups.length == 0) {<a name="line.464"></a>
 473 <span class="sourceLineNo">465</span>          outputFs.delete(tableDir, true);<a name="line.465"></a>
 474 <span class="sourceLineNo">466</span>          LOG.debug(tableDir.toString() + " is empty, remove it.");<a name="line.466"></a>
 475 <span class="sourceLineNo">467</span>        }<a name="line.467"></a>
 476 <span class="sourceLineNo">468</span>      }<a name="line.468"></a>
 477 <span class="sourceLineNo">469</span>      outputFs.delete(new Path(targetDir, backupInfo.getBackupId()), true);<a name="line.469"></a>
 478 <span class="sourceLineNo">470</span>    } catch (IOException e1) {<a name="line.470"></a>
 479 <span class="sourceLineNo">471</span>      LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " at "<a name="line.471"></a>
 480 <span class="sourceLineNo">472</span>          + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + ".");<a name="line.472"></a>
 481 <span class="sourceLineNo">473</span>    }<a name="line.473"></a>
 482 <span class="sourceLineNo">474</span>  }<a name="line.474"></a>
 483 <span class="sourceLineNo">475</span><a name="line.475"></a>
 484 <span class="sourceLineNo">476</span>  /**<a name="line.476"></a>
 485 <span class="sourceLineNo">477</span>   * Given the backup root dir, backup id and the table name, return the backup image location,<a name="line.477"></a>
 486 <span class="sourceLineNo">478</span>   * which is also where the backup manifest file is. return value look like:<a name="line.478"></a>
 487 <span class="sourceLineNo">479</span>   * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/backup_1396650096738/default/t1_dn/"<a name="line.479"></a>
 488 <span class="sourceLineNo">480</span>   * @param backupRootDir backup root directory<a name="line.480"></a>
 489 <span class="sourceLineNo">481</span>   * @param backupId backup id<a name="line.481"></a>
 490 <span class="sourceLineNo">482</span>   * @param tableName table name<a name="line.482"></a>
 491 <span class="sourceLineNo">483</span>   * @return backupPath String for the particular table<a name="line.483"></a>
 492 <span class="sourceLineNo">484</span>   */<a name="line.484"></a>
 493 <span class="sourceLineNo">485</span>  public static String getTableBackupDir(String backupRootDir, String backupId,<a name="line.485"></a>
 494 <span class="sourceLineNo">486</span>          TableName tableName) {<a name="line.486"></a>
 495 <span class="sourceLineNo">487</span>    return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR<a name="line.487"></a>
 496 <span class="sourceLineNo">488</span>        + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString()<a name="line.488"></a>
 497 <span class="sourceLineNo">489</span>        + Path.SEPARATOR;<a name="line.489"></a>
 498 <span class="sourceLineNo">490</span>  }<a name="line.490"></a>
 499 <span class="sourceLineNo">491</span><a name="line.491"></a>
 500 <span class="sourceLineNo">492</span>  /**<a name="line.492"></a>
 501 <span class="sourceLineNo">493</span>   * Sort history list by start time in descending order.<a name="line.493"></a>
 502 <span class="sourceLineNo">494</span>   * @param historyList history list<a name="line.494"></a>
 503 <span class="sourceLineNo">495</span>   * @return sorted list of BackupCompleteData<a name="line.495"></a>
 504 <span class="sourceLineNo">496</span>   */<a name="line.496"></a>
 505 <span class="sourceLineNo">497</span>  public static ArrayList&lt;BackupInfo&gt; sortHistoryListDesc(ArrayList&lt;BackupInfo&gt; historyList) {<a name="line.497"></a>
 506 <span class="sourceLineNo">498</span>    ArrayList&lt;BackupInfo&gt; list = new ArrayList&lt;&gt;();<a name="line.498"></a>
 507 <span class="sourceLineNo">499</span>    TreeMap&lt;String, BackupInfo&gt; map = new TreeMap&lt;&gt;();<a name="line.499"></a>
 508 <span class="sourceLineNo">500</span>    for (BackupInfo h : historyList) {<a name="line.500"></a>
 509 <span class="sourceLineNo">501</span>      map.put(Long.toString(h.getStartTs()), h);<a name="line.501"></a>
 510 <span class="sourceLineNo">502</span>    }<a name="line.502"></a>
 511 <span class="sourceLineNo">503</span>    Iterator&lt;String&gt; i = map.descendingKeySet().iterator();<a name="line.503"></a>
 512 <span class="sourceLineNo">504</span>    while (i.hasNext()) {<a name="line.504"></a>
 513 <span class="sourceLineNo">505</span>      list.add(map.get(i.next()));<a name="line.505"></a>
 514 <span class="sourceLineNo">506</span>    }<a name="line.506"></a>
 515 <span class="sourceLineNo">507</span>    return list;<a name="line.507"></a>
 516 <span class="sourceLineNo">508</span>  }<a name="line.508"></a>
 517 <span class="sourceLineNo">509</span><a name="line.509"></a>
 518 <span class="sourceLineNo">510</span>  /**<a name="line.510"></a>
 519 <span class="sourceLineNo">511</span>   * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates<a name="line.511"></a>
 520 <span class="sourceLineNo">512</span>   * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and<a name="line.512"></a>
 521 <span class="sourceLineNo">513</span>   * return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException.<a name="line.513"></a>
 522 <span class="sourceLineNo">514</span>   * @param fs file system<a name="line.514"></a>
 523 <span class="sourceLineNo">515</span>   * @param dir directory<a name="line.515"></a>
 524 <span class="sourceLineNo">516</span>   * @param filter path filter<a name="line.516"></a>
 525 <span class="sourceLineNo">517</span>   * @return null if dir is empty or doesn't exist, otherwise FileStatus array<a name="line.517"></a>
 526 <span class="sourceLineNo">518</span>   */<a name="line.518"></a>
 527 <span class="sourceLineNo">519</span>  public static FileStatus[] listStatus(final FileSystem fs, final Path dir,<a name="line.519"></a>
 528 <span class="sourceLineNo">520</span>          final PathFilter filter) throws IOException {<a name="line.520"></a>
 529 <span class="sourceLineNo">521</span>    FileStatus[] status = null;<a name="line.521"></a>
 530 <span class="sourceLineNo">522</span>    try {<a name="line.522"></a>
 531 <span class="sourceLineNo">523</span>      status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);<a name="line.523"></a>
 532 <span class="sourceLineNo">524</span>    } catch (FileNotFoundException fnfe) {<a name="line.524"></a>
 533 <span class="sourceLineNo">525</span>      // if directory doesn't exist, return null<a name="line.525"></a>
 534 <span class="sourceLineNo">526</span>      if (LOG.isTraceEnabled()) {<a name="line.526"></a>
 535 <span class="sourceLineNo">527</span>        LOG.trace(dir + " doesn't exist");<a name="line.527"></a>
 536 <span class="sourceLineNo">528</span>      }<a name="line.528"></a>
 537 <span class="sourceLineNo">529</span>    }<a name="line.529"></a>
 538 <span class="sourceLineNo">530</span><a name="line.530"></a>
 539 <span class="sourceLineNo">531</span>    if (status == null || status.length &lt; 1) {<a name="line.531"></a>
 540 <span class="sourceLineNo">532</span>      return null;<a name="line.532"></a>
 541 <span class="sourceLineNo">533</span>    }<a name="line.533"></a>
 542 <span class="sourceLineNo">534</span><a name="line.534"></a>
 543 <span class="sourceLineNo">535</span>    return status;<a name="line.535"></a>
 544 <span class="sourceLineNo">536</span>  }<a name="line.536"></a>
 545 <span class="sourceLineNo">537</span><a name="line.537"></a>
 546 <span class="sourceLineNo">538</span>  /**<a name="line.538"></a>
 547 <span class="sourceLineNo">539</span>   * Return the 'path' component of a Path. In Hadoop, Path is an URI. This method returns the<a name="line.539"></a>
 548 <span class="sourceLineNo">540</span>   * 'path' component of a Path's URI: e.g. If a Path is<a name="line.540"></a>
 549 <span class="sourceLineNo">541</span>   * &lt;code&gt;hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir&lt;/code&gt;, this method returns<a name="line.541"></a>
 550 <span class="sourceLineNo">542</span>   * &lt;code&gt;/hbase_trunk/TestTable/compaction.dir&lt;/code&gt;. This method is useful if you want to print<a name="line.542"></a>
 551 <span class="sourceLineNo">543</span>   * out a Path without qualifying Filesystem instance.<a name="line.543"></a>
 552 <span class="sourceLineNo">544</span>   * @param p file system Path whose 'path' component we are to return.<a name="line.544"></a>
 553 <span class="sourceLineNo">545</span>   * @return Path portion of the Filesystem<a name="line.545"></a>
 554 <span class="sourceLineNo">546</span>   */<a name="line.546"></a>
 555 <span class="sourceLineNo">547</span>  public static String getPath(Path p) {<a name="line.547"></a>
 556 <span class="sourceLineNo">548</span>    return p.toUri().getPath();<a name="line.548"></a>
 557 <span class="sourceLineNo">549</span>  }<a name="line.549"></a>
 558 <span class="sourceLineNo">550</span><a name="line.550"></a>
 559 <span class="sourceLineNo">551</span>  /**<a name="line.551"></a>
 560 <span class="sourceLineNo">552</span>   * Given the backup root dir and the backup id, return the log file location for an incremental<a name="line.552"></a>
 561 <span class="sourceLineNo">553</span>   * backup.<a name="line.553"></a>
 562 <span class="sourceLineNo">554</span>   * @param backupRootDir backup root directory<a name="line.554"></a>
 563 <span class="sourceLineNo">555</span>   * @param backupId backup id<a name="line.555"></a>
 564 <span class="sourceLineNo">556</span>   * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738"<a name="line.556"></a>
 565 <span class="sourceLineNo">557</span>   */<a name="line.557"></a>
 566 <span class="sourceLineNo">558</span>  public static String getLogBackupDir(String backupRootDir, String backupId) {<a name="line.558"></a>
 567 <span class="sourceLineNo">559</span>    return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR<a name="line.559"></a>
 568 <span class="sourceLineNo">560</span>        + HConstants.HREGION_LOGDIR_NAME;<a name="line.560"></a>
 569 <span class="sourceLineNo">561</span>  }<a name="line.561"></a>
 570 <span class="sourceLineNo">562</span><a name="line.562"></a>
 571 <span class="sourceLineNo">563</span>  private static List&lt;BackupInfo&gt; getHistory(Configuration conf, Path backupRootPath)<a name="line.563"></a>
 572 <span class="sourceLineNo">564</span>      throws IOException {<a name="line.564"></a>
 573 <span class="sourceLineNo">565</span>    // Get all (n) history from backup root destination<a name="line.565"></a>
 574 <span class="sourceLineNo">566</span>    FileSystem fs = FileSystem.get(conf);<a name="line.566"></a>
 575 <span class="sourceLineNo">567</span>    RemoteIterator&lt;LocatedFileStatus&gt; it = fs.listLocatedStatus(backupRootPath);<a name="line.567"></a>
 576 <span class="sourceLineNo">568</span><a name="line.568"></a>
 577 <span class="sourceLineNo">569</span>    List&lt;BackupInfo&gt; infos = new ArrayList&lt;&gt;();<a name="line.569"></a>
 578 <span class="sourceLineNo">570</span>    while (it.hasNext()) {<a name="line.570"></a>
 579 <span class="sourceLineNo">571</span>      LocatedFileStatus lfs = it.next();<a name="line.571"></a>
 580 <span class="sourceLineNo">572</span><a name="line.572"></a>
 581 <span class="sourceLineNo">573</span>      if (!lfs.isDirectory()) {<a name="line.573"></a>
 582 <span class="sourceLineNo">574</span>        continue;<a name="line.574"></a>
 583 <span class="sourceLineNo">575</span>      }<a name="line.575"></a>
 584 <span class="sourceLineNo">576</span><a name="line.576"></a>
 585 <span class="sourceLineNo">577</span>      String backupId = lfs.getPath().getName();<a name="line.577"></a>
 586 <span class="sourceLineNo">578</span>      try {<a name="line.578"></a>
 587 <span class="sourceLineNo">579</span>        BackupInfo info = loadBackupInfo(backupRootPath, backupId, fs);<a name="line.579"></a>
 588 <span class="sourceLineNo">580</span>        infos.add(info);<a name="line.580"></a>
 589 <span class="sourceLineNo">581</span>      } catch (IOException e) {<a name="line.581"></a>
 590 <span class="sourceLineNo">582</span>        LOG.error("Can not load backup info from: " + lfs.getPath(), e);<a name="line.582"></a>
 591 <span class="sourceLineNo">583</span>      }<a name="line.583"></a>
 592 <span class="sourceLineNo">584</span>    }<a name="line.584"></a>
 593 <span class="sourceLineNo">585</span>    // Sort<a name="line.585"></a>
 594 <span class="sourceLineNo">586</span>    Collections.sort(infos, new Comparator&lt;BackupInfo&gt;() {<a name="line.586"></a>
 595 <span class="sourceLineNo">587</span>      @Override<a name="line.587"></a>
 596 <span class="sourceLineNo">588</span>      public int compare(BackupInfo o1, BackupInfo o2) {<a name="line.588"></a>
 597 <span class="sourceLineNo">589</span>        long ts1 = getTimestamp(o1.getBackupId());<a name="line.589"></a>
 598 <span class="sourceLineNo">590</span>        long ts2 = getTimestamp(o2.getBackupId());<a name="line.590"></a>
 599 <span class="sourceLineNo">591</span><a name="line.591"></a>
 600 <span class="sourceLineNo">592</span>        if (ts1 == ts2) {<a name="line.592"></a>
 601 <span class="sourceLineNo">593</span>          return 0;<a name="line.593"></a>
 602 <span class="sourceLineNo">594</span>        }<a name="line.594"></a>
 603 <span class="sourceLineNo">595</span><a name="line.595"></a>
 604 <span class="sourceLineNo">596</span>        return ts1 &lt; ts2 ? 1 : -1;<a name="line.596"></a>
 605 <span class="sourceLineNo">597</span>      }<a name="line.597"></a>
 606 <span class="sourceLineNo">598</span><a name="line.598"></a>
 607 <span class="sourceLineNo">599</span>      private long getTimestamp(String backupId) {<a name="line.599"></a>
 608 <span class="sourceLineNo">600</span>        String[] split = backupId.split("_");<a name="line.600"></a>
 609 <span class="sourceLineNo">601</span>        return Long.parseLong(split[1]);<a name="line.601"></a>
 610 <span class="sourceLineNo">602</span>      }<a name="line.602"></a>
 611 <span class="sourceLineNo">603</span>    });<a name="line.603"></a>
 612 <span class="sourceLineNo">604</span>    return infos;<a name="line.604"></a>
 613 <span class="sourceLineNo">605</span>  }<a name="line.605"></a>
 614 <span class="sourceLineNo">606</span><a name="line.606"></a>
 615 <span class="sourceLineNo">607</span>  public static List&lt;BackupInfo&gt; getHistory(Configuration conf, int n, Path backupRootPath,<a name="line.607"></a>
 616 <span class="sourceLineNo">608</span>      BackupInfo.Filter... filters) throws IOException {<a name="line.608"></a>
 617 <span class="sourceLineNo">609</span>    List&lt;BackupInfo&gt; infos = getHistory(conf, backupRootPath);<a name="line.609"></a>
 618 <span class="sourceLineNo">610</span>    List&lt;BackupInfo&gt; ret = new ArrayList&lt;&gt;();<a name="line.610"></a>
 619 <span class="sourceLineNo">611</span>    for (BackupInfo info : infos) {<a name="line.611"></a>
 620 <span class="sourceLineNo">612</span>      if (ret.size() == n) {<a name="line.612"></a>
 621 <span class="sourceLineNo">613</span>        break;<a name="line.613"></a>
 622 <span class="sourceLineNo">614</span>      }<a name="line.614"></a>
 623 <span class="sourceLineNo">615</span>      boolean passed = true;<a name="line.615"></a>
 624 <span class="sourceLineNo">616</span>      for (int i = 0; i &lt; filters.length; i++) {<a name="line.616"></a>
 625 <span class="sourceLineNo">617</span>        if (!filters[i].apply(info)) {<a name="line.617"></a>
 626 <span class="sourceLineNo">618</span>          passed = false;<a name="line.618"></a>
 627 <span class="sourceLineNo">619</span>          break;<a name="line.619"></a>
 628 <span class="sourceLineNo">620</span>        }<a name="line.620"></a>
 629 <span class="sourceLineNo">621</span>      }<a name="line.621"></a>
 630 <span class="sourceLineNo">622</span>      if (passed) {<a name="line.622"></a>
 631 <span class="sourceLineNo">623</span>        ret.add(info);<a name="line.623"></a>
 632 <span class="sourceLineNo">624</span>      }<a name="line.624"></a>
 633 <span class="sourceLineNo">625</span>    }<a name="line.625"></a>
 634 <span class="sourceLineNo">626</span>    return ret;<a name="line.626"></a>
 635 <span class="sourceLineNo">627</span>  }<a name="line.627"></a>
 636 <span class="sourceLineNo">628</span><a name="line.628"></a>
 637 <span class="sourceLineNo">629</span>  public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs)<a name="line.629"></a>
 638 <span class="sourceLineNo">630</span>      throws IOException {<a name="line.630"></a>
 639 <span class="sourceLineNo">631</span>    Path backupPath = new Path(backupRootPath, backupId);<a name="line.631"></a>
 640 <span class="sourceLineNo">632</span><a name="line.632"></a>
 641 <span class="sourceLineNo">633</span>    RemoteIterator&lt;LocatedFileStatus&gt; it = fs.listFiles(backupPath, true);<a name="line.633"></a>
 642 <span class="sourceLineNo">634</span>    while (it.hasNext()) {<a name="line.634"></a>
 643 <span class="sourceLineNo">635</span>      LocatedFileStatus lfs = it.next();<a name="line.635"></a>
 644 <span class="sourceLineNo">636</span>      if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) {<a name="line.636"></a>
 645 <span class="sourceLineNo">637</span>        // Load BackupManifest<a name="line.637"></a>
 646 <span class="sourceLineNo">638</span>        BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent());<a name="line.638"></a>
 647 <span class="sourceLineNo">639</span>        BackupInfo info = manifest.toBackupInfo();<a name="line.639"></a>
 648 <span class="sourceLineNo">640</span>        return info;<a name="line.640"></a>
 649 <span class="sourceLineNo">641</span>      }<a name="line.641"></a>
 650 <span class="sourceLineNo">642</span>    }<a name="line.642"></a>
 651 <span class="sourceLineNo">643</span>    return null;<a name="line.643"></a>
 652 <span class="sourceLineNo">644</span>  }<a name="line.644"></a>
 653 <span class="sourceLineNo">645</span><a name="line.645"></a>
 654 <span class="sourceLineNo">646</span>  /**<a name="line.646"></a>
 655 <span class="sourceLineNo">647</span>   * Create restore request.<a name="line.647"></a>
 656 <span class="sourceLineNo">648</span>   * @param backupRootDir backup root dir<a name="line.648"></a>
 657 <span class="sourceLineNo">649</span>   * @param backupId backup id<a name="line.649"></a>
 658 <span class="sourceLineNo">650</span>   * @param check check only<a name="line.650"></a>
 659 <span class="sourceLineNo">651</span>   * @param fromTables table list from<a name="line.651"></a>
 660 <span class="sourceLineNo">652</span>   * @param toTables table list to<a name="line.652"></a>
 661 <span class="sourceLineNo">653</span>   * @param isOverwrite overwrite data<a name="line.653"></a>
 662 <span class="sourceLineNo">654</span>   * @return request obkect<a name="line.654"></a>
 663 <span class="sourceLineNo">655</span>   */<a name="line.655"></a>
 664 <span class="sourceLineNo">656</span>  public static RestoreRequest createRestoreRequest(String backupRootDir, String backupId,<a name="line.656"></a>
 665 <span class="sourceLineNo">657</span>      boolean check, TableName[] fromTables, TableName[] toTables, boolean isOverwrite) {<a name="line.657"></a>
 666 <span class="sourceLineNo">658</span>    RestoreRequest.Builder builder = new RestoreRequest.Builder();<a name="line.658"></a>
 667 <span class="sourceLineNo">659</span>    RestoreRequest request =<a name="line.659"></a>
 668 <span class="sourceLineNo">660</span>        builder.withBackupRootDir(backupRootDir).withBackupId(backupId).withCheck(check)<a name="line.660"></a>
 669 <span class="sourceLineNo">661</span>            .withFromTables(fromTables).withToTables(toTables).withOvewrite(isOverwrite).build();<a name="line.661"></a>
 670 <span class="sourceLineNo">662</span>    return request;<a name="line.662"></a>
 671 <span class="sourceLineNo">663</span>  }<a name="line.663"></a>
 672 <span class="sourceLineNo">664</span><a name="line.664"></a>
 673 <span class="sourceLineNo">665</span>  public static boolean validate(HashMap&lt;TableName, BackupManifest&gt; backupManifestMap,<a name="line.665"></a>
 674 <span class="sourceLineNo">666</span>      Configuration conf) throws IOException {<a name="line.666"></a>
 675 <span class="sourceLineNo">667</span>    boolean isValid = true;<a name="line.667"></a>
 676 <span class="sourceLineNo">668</span><a name="line.668"></a>
 677 <span class="sourceLineNo">669</span>    for (Entry&lt;TableName, BackupManifest&gt; manifestEntry : backupManifestMap.entrySet()) {<a name="line.669"></a>
 678 <span class="sourceLineNo">670</span>      TableName table = manifestEntry.getKey();<a name="line.670"></a>
 679 <span class="sourceLineNo">671</span>      TreeSet&lt;BackupImage&gt; imageSet = new TreeSet&lt;&gt;();<a name="line.671"></a>
 680 <span class="sourceLineNo">672</span><a name="line.672"></a>
 681 <span class="sourceLineNo">673</span>      ArrayList&lt;BackupImage&gt; depList = manifestEntry.getValue().getDependentListByTable(table);<a name="line.673"></a>
 682 <span class="sourceLineNo">674</span>      if (depList != null &amp;&amp; !depList.isEmpty()) {<a name="line.674"></a>
 683 <span class="sourceLineNo">675</span>        imageSet.addAll(depList);<a name="line.675"></a>
 684 <span class="sourceLineNo">676</span>      }<a name="line.676"></a>
 685 <span class="sourceLineNo">677</span><a name="line.677"></a>
 686 <span class="sourceLineNo">678</span>      LOG.info("Dependent image(s) from old to new:");<a name="line.678"></a>
 687 <span class="sourceLineNo">679</span>      for (BackupImage image : imageSet) {<a name="line.679"></a>
 688 <span class="sourceLineNo">680</span>        String imageDir =<a name="line.680"></a>
 689 <span class="sourceLineNo">681</span>            HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(), table);<a name="line.681"></a>
 690 <span class="sourceLineNo">682</span>        if (!BackupUtils.checkPathExist(imageDir, conf)) {<a name="line.682"></a>
 691 <span class="sourceLineNo">683</span>          LOG.error("ERROR: backup image does not exist: " + imageDir);<a name="line.683"></a>
 692 <span class="sourceLineNo">684</span>          isValid = false;<a name="line.684"></a>
 693 <span class="sourceLineNo">685</span>          break;<a name="line.685"></a>
 694 <span class="sourceLineNo">686</span>        }<a name="line.686"></a>
 695 <span class="sourceLineNo">687</span>        LOG.info("Backup image: " + image.getBackupId() + " for '" + table + "' is available");<a name="line.687"></a>
 696 <span class="sourceLineNo">688</span>      }<a name="line.688"></a>
 697 <span class="sourceLineNo">689</span>    }<a name="line.689"></a>
 698 <span class="sourceLineNo">690</span>    return isValid;<a name="line.690"></a>
 699 <span class="sourceLineNo">691</span>  }<a name="line.691"></a>
 700 <span class="sourceLineNo">692</span><a name="line.692"></a>
 701 <span class="sourceLineNo">693</span>  public static Path getBulkOutputDir(String tableName, Configuration conf, boolean deleteOnExit)<a name="line.693"></a>
 702 <span class="sourceLineNo">694</span>      throws IOException {<a name="line.694"></a>
 703 <span class="sourceLineNo">695</span>    FileSystem fs = FileSystem.get(conf);<a name="line.695"></a>
 704 <span class="sourceLineNo">696</span>    String tmp = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY,<a name="line.696"></a>
 705 <span class="sourceLineNo">697</span>            HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY);<a name="line.697"></a>
 706 <span class="sourceLineNo">698</span>    Path path =<a name="line.698"></a>
 707 <span class="sourceLineNo">699</span>        new Path(tmp + Path.SEPARATOR + "bulk_output-" + tableName + "-"<a name="line.699"></a>
 708 <span class="sourceLineNo">700</span>            + EnvironmentEdgeManager.currentTime());<a name="line.700"></a>
 709 <span class="sourceLineNo">701</span>    if (deleteOnExit) {<a name="line.701"></a>
 710 <span class="sourceLineNo">702</span>      fs.deleteOnExit(path);<a name="line.702"></a>
 711 <span class="sourceLineNo">703</span>    }<a name="line.703"></a>
 712 <span class="sourceLineNo">704</span>    return path;<a name="line.704"></a>
 713 <span class="sourceLineNo">705</span>  }<a name="line.705"></a>
 714 <span class="sourceLineNo">706</span><a name="line.706"></a>
 715 <span class="sourceLineNo">707</span>  public static Path getBulkOutputDir(String tableName, Configuration conf) throws IOException {<a name="line.707"></a>
 716 <span class="sourceLineNo">708</span>    return getBulkOutputDir(tableName, conf, true);<a name="line.708"></a>
 717 <span class="sourceLineNo">709</span>  }<a name="line.709"></a>
 718 <span class="sourceLineNo">710</span><a name="line.710"></a>
 719 <span class="sourceLineNo">711</span>  public static String getFileNameCompatibleString(TableName table) {<a name="line.711"></a>
 720 <span class="sourceLineNo">712</span>    return table.getNamespaceAsString() + "-" + table.getQualifierAsString();<a name="line.712"></a>
 721 <span class="sourceLineNo">713</span>  }<a name="line.713"></a>
 722 <span class="sourceLineNo">714</span><a name="line.714"></a>
 723 <span class="sourceLineNo">715</span>  public static boolean failed(int result) {<a name="line.715"></a>
 724 <span class="sourceLineNo">716</span>    return result != 0;<a name="line.716"></a>
 725 <span class="sourceLineNo">717</span>  }<a name="line.717"></a>
 726 <span class="sourceLineNo">718</span><a name="line.718"></a>
 727 <span class="sourceLineNo">719</span>  public static boolean succeeded(int result) {<a name="line.719"></a>
 728 <span class="sourceLineNo">720</span>    return result == 0;<a name="line.720"></a>
 729 <span class="sourceLineNo">721</span>  }<a name="line.721"></a>
 730 <span class="sourceLineNo">722</span><a name="line.722"></a>
 731 <span class="sourceLineNo">723</span>  public static LoadIncrementalHFiles createLoader(Configuration config) throws IOException {<a name="line.723"></a>
 732 <span class="sourceLineNo">724</span>    // set configuration for restore:<a name="line.724"></a>
 733 <span class="sourceLineNo">725</span>    // LoadIncrementalHFile needs more time<a name="line.725"></a>
 734 <span class="sourceLineNo">726</span>    // &lt;name&gt;hbase.rpc.timeout&lt;/name&gt; &lt;value&gt;600000&lt;/value&gt;<a name="line.726"></a>
 735 <span class="sourceLineNo">727</span>    // calculates<a name="line.727"></a>
 736 <span class="sourceLineNo">728</span>    Configuration conf = new Configuration(config);<a name="line.728"></a>
 737 <span class="sourceLineNo">729</span>    conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, MILLISEC_IN_HOUR);<a name="line.729"></a>
 738 <span class="sourceLineNo">730</span><a name="line.730"></a>
 739 <span class="sourceLineNo">731</span>    // By default, it is 32 and loader will fail if # of files in any region exceed this<a name="line.731"></a>
 740 <span class="sourceLineNo">732</span>    // limit. Bad for snapshot restore.<a name="line.732"></a>
 741 <span class="sourceLineNo">733</span>    conf.setInt(LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, Integer.MAX_VALUE);<a name="line.733"></a>
 742 <span class="sourceLineNo">734</span>    conf.set(LoadIncrementalHFiles.IGNORE_UNMATCHED_CF_CONF_KEY, "yes");<a name="line.734"></a>
 743 <span class="sourceLineNo">735</span>    LoadIncrementalHFiles loader;<a name="line.735"></a>
 744 <span class="sourceLineNo">736</span>    try {<a name="line.736"></a>
 745 <span class="sourceLineNo">737</span>      loader = new LoadIncrementalHFiles(conf);<a name="line.737"></a>
 746 <span class="sourceLineNo">738</span>    } catch (Exception e) {<a name="line.738"></a>
 747 <span class="sourceLineNo">739</span>      throw new IOException(e);<a name="line.739"></a>
 748 <span class="sourceLineNo">740</span>    }<a name="line.740"></a>
 749 <span class="sourceLineNo">741</span>    return loader;<a name="line.741"></a>
 750 <span class="sourceLineNo">742</span>  }<a name="line.742"></a>
 751 <span class="sourceLineNo">743</span><a name="line.743"></a>
 752 <span class="sourceLineNo">744</span>  public static String findMostRecentBackupId(String[] backupIds) {<a name="line.744"></a>
 753 <span class="sourceLineNo">745</span>    long recentTimestamp = Long.MIN_VALUE;<a name="line.745"></a>
 754 <span class="sourceLineNo">746</span>    for (String backupId : backupIds) {<a name="line.746"></a>
 755 <span class="sourceLineNo">747</span>      long ts = Long.parseLong(backupId.split("_")[1]);<a name="line.747"></a>
 756 <span class="sourceLineNo">748</span>      if (ts &gt; recentTimestamp) {<a name="line.748"></a>
 757 <span class="sourceLineNo">749</span>        recentTimestamp = ts;<a name="line.749"></a>
 758 <span class="sourceLineNo">750</span>      }<a name="line.750"></a>
 759 <span class="sourceLineNo">751</span>    }<a name="line.751"></a>
 760 <span class="sourceLineNo">752</span>    return BackupRestoreConstants.BACKUPID_PREFIX + recentTimestamp;<a name="line.752"></a>
 761 <span class="sourceLineNo">753</span>  }<a name="line.753"></a>
 762 <span class="sourceLineNo">754</span><a name="line.754"></a>
 763 <span class="sourceLineNo">755</span>}<a name="line.755"></a>
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824 </pre>
 825 </div>
 826 </body>
 827 </html>