Line data Source code
1 : // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file. See the AUTHORS file for names of contributors.
4 : //
5 : // We recover the contents of the descriptor from the other files we find.
6 : // (1) Any log files are first converted to tables
7 : // (2) We scan every table to compute
8 : // (a) smallest/largest for the table
9 : // (b) largest sequence number in the table
10 : // (3) We generate descriptor contents:
11 : // - log number is set to zero
12 : // - next-file-number is set to 1 + largest file number we found
13 : // - last-sequence-number is set to largest sequence# found across
14 : // all tables (see 2c)
15 : // - compaction pointers are cleared
16 : // - every table file is added at level 0
17 : //
18 : // Possible optimization 1:
19 : // (a) Compute total size and use to pick appropriate max-level M
20 : // (b) Sort tables by largest sequence# in the table
21 : // (c) For each table: if it overlaps earlier table, place in level-0,
22 : // else place in level-M.
23 : // Possible optimization 2:
24 : // Store per-table metadata (smallest, largest, largest-seq#, ...)
25 : // in the table's meta section to speed up ScanTable.
26 :
27 : #include "db/builder.h"
28 : #include "db/db_impl.h"
29 : #include "db/dbformat.h"
30 : #include "db/filename.h"
31 : #include "db/log_reader.h"
32 : #include "db/log_writer.h"
33 : #include "db/memtable.h"
34 : #include "db/table_cache.h"
35 : #include "db/version_edit.h"
36 : #include "db/write_batch_internal.h"
37 : #include "leveldb/comparator.h"
38 : #include "leveldb/db.h"
39 : #include "leveldb/env.h"
40 :
41 : namespace leveldb {
42 :
43 : namespace {
44 :
45 : class Repairer {
46 : public:
47 0 : Repairer(const std::string& dbname, const Options& options)
48 : : dbname_(dbname),
49 : env_(options.env),
50 : icmp_(options.comparator),
51 : ipolicy_(options.filter_policy),
52 0 : options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
53 0 : owns_info_log_(options_.info_log != options.info_log),
54 0 : owns_cache_(options_.block_cache != options.block_cache),
55 0 : next_file_number_(1) {
56 : // TableCache can be small since we expect each table to be opened once.
57 0 : table_cache_ = new TableCache(dbname_, &options_, 10);
58 0 : }
59 :
60 0 : ~Repairer() {
61 0 : delete table_cache_;
62 0 : if (owns_info_log_) {
63 0 : delete options_.info_log;
64 : }
65 0 : if (owns_cache_) {
66 0 : delete options_.block_cache;
67 : }
68 0 : }
69 :
70 0 : Status Run() {
71 0 : Status status = FindFiles();
72 0 : if (status.ok()) {
73 0 : ConvertLogFilesToTables();
74 0 : ExtractMetaData();
75 0 : status = WriteDescriptor();
76 : }
77 0 : if (status.ok()) {
78 : unsigned long long bytes = 0;
79 0 : for (size_t i = 0; i < tables_.size(); i++) {
80 0 : bytes += tables_[i].meta.file_size;
81 : }
82 : Log(options_.info_log,
83 : "**** Repaired leveldb %s; "
84 : "recovered %d files; %llu bytes. "
85 : "Some data may have been lost. "
86 : "****",
87 : dbname_.c_str(),
88 0 : static_cast<int>(tables_.size()),
89 0 : bytes);
90 : }
91 0 : return status;
92 : }
93 :
94 : private:
95 0 : struct TableInfo {
96 : FileMetaData meta;
97 : SequenceNumber max_sequence;
98 : };
99 :
100 : std::string const dbname_;
101 : Env* const env_;
102 : InternalKeyComparator const icmp_;
103 : InternalFilterPolicy const ipolicy_;
104 : Options const options_;
105 : bool owns_info_log_;
106 : bool owns_cache_;
107 : TableCache* table_cache_;
108 : VersionEdit edit_;
109 :
110 : std::vector<std::string> manifests_;
111 : std::vector<uint64_t> table_numbers_;
112 : std::vector<uint64_t> logs_;
113 : std::vector<TableInfo> tables_;
114 : uint64_t next_file_number_;
115 :
116 0 : Status FindFiles() {
117 : std::vector<std::string> filenames;
118 0 : Status status = env_->GetChildren(dbname_, &filenames);
119 0 : if (!status.ok()) {
120 : return status;
121 : }
122 0 : if (filenames.empty()) {
123 0 : return Status::IOError(dbname_, "repair found no files");
124 : }
125 :
126 : uint64_t number;
127 : FileType type;
128 0 : for (size_t i = 0; i < filenames.size(); i++) {
129 0 : if (ParseFileName(filenames[i], &number, &type)) {
130 0 : if (type == kDescriptorFile) {
131 0 : manifests_.push_back(filenames[i]);
132 : } else {
133 0 : if (number + 1 > next_file_number_) {
134 0 : next_file_number_ = number + 1;
135 : }
136 0 : if (type == kLogFile) {
137 0 : logs_.push_back(number);
138 0 : } else if (type == kTableFile) {
139 0 : table_numbers_.push_back(number);
140 : } else {
141 : // Ignore other files
142 : }
143 : }
144 : }
145 : }
146 0 : return status;
147 : }
148 :
149 0 : void ConvertLogFilesToTables() {
150 0 : for (size_t i = 0; i < logs_.size(); i++) {
151 0 : std::string logname = LogFileName(dbname_, logs_[i]);
152 0 : Status status = ConvertLogToTable(logs_[i]);
153 0 : if (!status.ok()) {
154 : Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
155 0 : (unsigned long long) logs_[i],
156 0 : status.ToString().c_str());
157 : }
158 0 : ArchiveFile(logname);
159 : }
160 0 : }
161 :
162 0 : Status ConvertLogToTable(uint64_t log) {
163 0 : struct LogReporter : public log::Reader::Reporter {
164 : Env* env;
165 : Logger* info_log;
166 : uint64_t lognum;
167 0 : virtual void Corruption(size_t bytes, const Status& s) {
168 : // We print error messages for corruption, but continue repairing.
169 : Log(info_log, "Log #%llu: dropping %d bytes; %s",
170 : (unsigned long long) lognum,
171 : static_cast<int>(bytes),
172 0 : s.ToString().c_str());
173 0 : }
174 : };
175 :
176 : // Open the log file
177 0 : std::string logname = LogFileName(dbname_, log);
178 : SequentialFile* lfile;
179 0 : Status status = env_->NewSequentialFile(logname, &lfile);
180 0 : if (!status.ok()) {
181 : return status;
182 : }
183 :
184 : // Create the log reader.
185 : LogReporter reporter;
186 0 : reporter.env = env_;
187 0 : reporter.info_log = options_.info_log;
188 0 : reporter.lognum = log;
189 : // We intentionally make log::Reader do checksumming so that
190 : // corruptions cause entire commits to be skipped instead of
191 : // propagating bad information (like overly large sequence
192 : // numbers).
193 : log::Reader reader(lfile, &reporter, false/*do not checksum*/,
194 0 : 0/*initial_offset*/);
195 :
196 : // Read all the records and add to a memtable
197 : std::string scratch;
198 : Slice record;
199 0 : WriteBatch batch;
200 0 : MemTable* mem = new MemTable(icmp_);
201 0 : mem->Ref();
202 0 : int counter = 0;
203 0 : while (reader.ReadRecord(&record, &scratch)) {
204 0 : if (record.size() < 12) {
205 : reporter.Corruption(
206 0 : record.size(), Status::Corruption("log record too small"));
207 0 : continue;
208 : }
209 0 : WriteBatchInternal::SetContents(&batch, record);
210 0 : status = WriteBatchInternal::InsertInto(&batch, mem);
211 0 : if (status.ok()) {
212 0 : counter += WriteBatchInternal::Count(&batch);
213 : } else {
214 : Log(options_.info_log, "Log #%llu: ignoring %s",
215 : (unsigned long long) log,
216 0 : status.ToString().c_str());
217 0 : status = Status::OK(); // Keep going with rest of file
218 : }
219 : }
220 0 : delete lfile;
221 :
222 : // Do not record a version edit for this conversion to a Table
223 : // since ExtractMetaData() will also generate edits.
224 0 : FileMetaData meta;
225 0 : meta.number = next_file_number_++;
226 0 : Iterator* iter = mem->NewIterator();
227 0 : status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
228 0 : delete iter;
229 0 : mem->Unref();
230 0 : mem = NULL;
231 0 : if (status.ok()) {
232 0 : if (meta.file_size > 0) {
233 0 : table_numbers_.push_back(meta.number);
234 : }
235 : }
236 : Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
237 : (unsigned long long) log,
238 : counter,
239 : (unsigned long long) meta.number,
240 0 : status.ToString().c_str());
241 : return status;
242 : }
243 :
244 0 : void ExtractMetaData() {
245 0 : for (size_t i = 0; i < table_numbers_.size(); i++) {
246 0 : ScanTable(table_numbers_[i]);
247 : }
248 0 : }
249 :
250 0 : Iterator* NewTableIterator(const FileMetaData& meta) {
251 : // Same as compaction iterators: if paranoid_checks are on, turn
252 : // on checksum verification.
253 : ReadOptions r;
254 0 : r.verify_checksums = options_.paranoid_checks;
255 0 : return table_cache_->NewIterator(r, meta.number, meta.file_size);
256 : }
257 :
258 0 : void ScanTable(uint64_t number) {
259 : TableInfo t;
260 0 : t.meta.number = number;
261 0 : std::string fname = TableFileName(dbname_, number);
262 0 : Status status = env_->GetFileSize(fname, &t.meta.file_size);
263 0 : if (!status.ok()) {
264 : // Try alternate file name.
265 0 : fname = SSTTableFileName(dbname_, number);
266 0 : Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
267 0 : if (s2.ok()) {
268 0 : status = Status::OK();
269 : }
270 : }
271 0 : if (!status.ok()) {
272 0 : ArchiveFile(TableFileName(dbname_, number));
273 0 : ArchiveFile(SSTTableFileName(dbname_, number));
274 : Log(options_.info_log, "Table #%llu: dropped: %s",
275 : (unsigned long long) t.meta.number,
276 0 : status.ToString().c_str());
277 0 : return;
278 : }
279 :
280 : // Extract metadata by scanning through table.
281 0 : int counter = 0;
282 0 : Iterator* iter = NewTableIterator(t.meta);
283 0 : bool empty = true;
284 : ParsedInternalKey parsed;
285 0 : t.max_sequence = 0;
286 0 : for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
287 0 : Slice key = iter->key();
288 0 : if (!ParseInternalKey(key, &parsed)) {
289 : Log(options_.info_log, "Table #%llu: unparsable key %s",
290 : (unsigned long long) t.meta.number,
291 0 : EscapeString(key).c_str());
292 0 : continue;
293 : }
294 :
295 0 : counter++;
296 0 : if (empty) {
297 0 : empty = false;
298 : t.meta.smallest.DecodeFrom(key);
299 : }
300 : t.meta.largest.DecodeFrom(key);
301 0 : if (parsed.sequence > t.max_sequence) {
302 0 : t.max_sequence = parsed.sequence;
303 : }
304 : }
305 0 : if (!iter->status().ok()) {
306 0 : status = iter->status();
307 : }
308 0 : delete iter;
309 : Log(options_.info_log, "Table #%llu: %d entries %s",
310 : (unsigned long long) t.meta.number,
311 : counter,
312 0 : status.ToString().c_str());
313 :
314 0 : if (status.ok()) {
315 0 : tables_.push_back(t);
316 : } else {
317 0 : RepairTable(fname, t); // RepairTable archives input file.
318 : }
319 : }
320 :
321 0 : void RepairTable(const std::string& src, TableInfo t) {
322 : // We will copy src contents to a new table and then rename the
323 : // new table over the source.
324 :
325 : // Create builder.
326 0 : std::string copy = TableFileName(dbname_, next_file_number_++);
327 : WritableFile* file;
328 0 : Status s = env_->NewWritableFile(copy, &file);
329 0 : if (!s.ok()) {
330 0 : return;
331 : }
332 0 : TableBuilder* builder = new TableBuilder(options_, file);
333 :
334 : // Copy data.
335 0 : Iterator* iter = NewTableIterator(t.meta);
336 0 : int counter = 0;
337 0 : for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
338 0 : builder->Add(iter->key(), iter->value());
339 0 : counter++;
340 : }
341 0 : delete iter;
342 :
343 0 : ArchiveFile(src);
344 0 : if (counter == 0) {
345 0 : builder->Abandon(); // Nothing to save
346 : } else {
347 0 : s = builder->Finish();
348 0 : if (s.ok()) {
349 0 : t.meta.file_size = builder->FileSize();
350 : }
351 : }
352 0 : delete builder;
353 0 : builder = NULL;
354 :
355 0 : if (s.ok()) {
356 0 : s = file->Close();
357 : }
358 0 : delete file;
359 0 : file = NULL;
360 :
361 0 : if (counter > 0 && s.ok()) {
362 0 : std::string orig = TableFileName(dbname_, t.meta.number);
363 0 : s = env_->RenameFile(copy, orig);
364 0 : if (s.ok()) {
365 : Log(options_.info_log, "Table #%llu: %d entries repaired",
366 0 : (unsigned long long) t.meta.number, counter);
367 0 : tables_.push_back(t);
368 : }
369 : }
370 0 : if (!s.ok()) {
371 0 : env_->DeleteFile(copy);
372 : }
373 : }
374 :
375 0 : Status WriteDescriptor() {
376 0 : std::string tmp = TempFileName(dbname_, 1);
377 : WritableFile* file;
378 0 : Status status = env_->NewWritableFile(tmp, &file);
379 0 : if (!status.ok()) {
380 : return status;
381 : }
382 :
383 : SequenceNumber max_sequence = 0;
384 0 : for (size_t i = 0; i < tables_.size(); i++) {
385 0 : if (max_sequence < tables_[i].max_sequence) {
386 0 : max_sequence = tables_[i].max_sequence;
387 : }
388 : }
389 :
390 0 : edit_.SetComparatorName(icmp_.user_comparator()->Name());
391 0 : edit_.SetLogNumber(0);
392 0 : edit_.SetNextFile(next_file_number_);
393 0 : edit_.SetLastSequence(max_sequence);
394 :
395 0 : for (size_t i = 0; i < tables_.size(); i++) {
396 : // TODO(opt): separate out into multiple levels
397 0 : const TableInfo& t = tables_[i];
398 : edit_.AddFile(0, t.meta.number, t.meta.file_size,
399 0 : t.meta.smallest, t.meta.largest);
400 : }
401 :
402 : //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
403 : {
404 0 : log::Writer log(file);
405 : std::string record;
406 0 : edit_.EncodeTo(&record);
407 0 : status = log.AddRecord(record);
408 : }
409 0 : if (status.ok()) {
410 0 : status = file->Close();
411 : }
412 0 : delete file;
413 0 : file = NULL;
414 :
415 0 : if (!status.ok()) {
416 0 : env_->DeleteFile(tmp);
417 : } else {
418 : // Discard older manifests
419 0 : for (size_t i = 0; i < manifests_.size(); i++) {
420 0 : ArchiveFile(dbname_ + "/" + manifests_[i]);
421 : }
422 :
423 : // Install new manifest
424 0 : status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
425 0 : if (status.ok()) {
426 0 : status = SetCurrentFile(env_, dbname_, 1);
427 : } else {
428 0 : env_->DeleteFile(tmp);
429 : }
430 : }
431 : return status;
432 : }
433 :
434 0 : void ArchiveFile(const std::string& fname) {
435 : // Move into another directory. E.g., for
436 : // dir/foo
437 : // rename to
438 : // dir/lost/foo
439 0 : const char* slash = strrchr(fname.c_str(), '/');
440 : std::string new_dir;
441 0 : if (slash != NULL) {
442 0 : new_dir.assign(fname.data(), slash - fname.data());
443 : }
444 0 : new_dir.append("/lost");
445 0 : env_->CreateDir(new_dir); // Ignore error
446 0 : std::string new_file = new_dir;
447 0 : new_file.append("/");
448 0 : new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
449 0 : Status s = env_->RenameFile(fname, new_file);
450 : Log(options_.info_log, "Archiving %s: %s\n",
451 0 : fname.c_str(), s.ToString().c_str());
452 0 : }
453 : };
454 : } // namespace
455 :
456 0 : Status RepairDB(const std::string& dbname, const Options& options) {
457 0 : Repairer repairer(dbname, options);
458 0 : return repairer.Run();
459 : }
460 :
461 : } // namespace leveldb
|
