1058{
1060
1062
1063 std::string default_db_type = "lmdb";
1064
1066 available_dbs = "available: " + available_dbs;
1067
1069
1071
1072 boost::filesystem::path output_file_path;
1073
1074 po::options_description desc_cmd_only("Command line options");
1075 po::options_description desc_cmd_sett("Command line options and settings options");
1077 "spent-output-db-dir", "Specify spent output database directory",
1078 get_default_db_path(),
1079 };
1082 "database", available_dbs.c_str(), default_db_type
1083 };
1089 "db-sync-mode"
1090 , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
1091 , "fast:1000"
1092 };
1095 const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {
"force-chain-reaction-pass",
"Run the chain reaction pass even if no new blockchain data was processed"};
1096
1109
1110 po::options_description desc_options("Allowed options");
1111 desc_options.add(desc_cmd_only).add(desc_cmd_sett);
1112
1113 po::positional_options_description positional_options;
1114 positional_options.add(arg_inputs.name, -1);
1115
1116 po::variables_map vm;
1118 {
1119 auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
1120 po::store(parser.run(), vm);
1121 po::notify(vm);
1122 return true;
1123 });
1124 if (! r)
1125 return 1;
1126
1128 {
1130 std::cout << desc_options << std::endl;
1131 return 1;
1132 }
1133
1137 else
1138 mlog_set_log(std::string(std::to_string(log_level) +
",bcutil:INFO").c_str());
1139
1141
1149 std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list);
1150
1153 {
1154 std::cerr << "Invalid database type: " << db_type << std::endl;
1155 return 1;
1156 }
1157
1159 if (!parse_db_sync_mode(db_sync_mode))
1160 {
1161 MERROR(
"Invalid db sync mode: " << db_sync_mode);
1162 return 1;
1163 }
1164
1166 if (inputs.empty())
1167 {
1169 return 1;
1170 }
1171
1172 const std::string cache_dir = (output_file_path / "spent-outputs-cache").string();
1173 init(cache_dir);
1174
1176
1177 size_t done = 0;
1178
1179 const uint64_t start_blackballed_outputs = get_num_spent_outputs();
1180
1182
1183 bool stop_requested = false;
1185 stop_requested = true;
1186 });
1187
1188 int dbr = resize_env(cache_dir.c_str());
1190
1191
1196 open_db(inputs[0], &env0, &txn0, &cur0, &dbi0);
1197
1198 if (!extra_spent_outputs.empty())
1199 {
1200 MINFO(
"Adding " << extra_spent_outputs.size() <<
" extra spent outputs");
1207
1208 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1209 for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs)
1210 {
1211 if (!is_output_spent(cur,
output_data(output.first, output.second)))
1212 {
1213 blackballs.push_back(output);
1214 if (add_spent_output(cur,
output_data(output.first, output.second)))
1215 inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra");
1216 }
1217 }
1218 if (!blackballs.empty())
1219 {
1220 ringdb.blackball(blackballs);
1221 blackballs.clear();
1222 }
1226 }
1227
1228 for (size_t n = 0; n < inputs.size(); ++n)
1229 {
1230 const std::string canonical = boost::filesystem::canonical(inputs[n]).string();
1231 uint64_t start_idx = get_processed_txidx(canonical);
1232 if (n > 0 && start_idx == 0)
1233 {
1234 start_idx = find_first_diverging_transaction(inputs[0], inputs[n]);
1235 LOG_PRINT_L0(
"First diverging transaction at " << start_idx);
1236 }
1237 LOG_PRINT_L0(
"Reading blockchain from " << inputs[n] <<
" from " << start_idx);
1244 size_t records = 0;
1245 const std::string filename = inputs[n];
1246 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1249 {
1250 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1251 const bool miner_tx = tx.
vin.size() == 1 && tx.
vin[0].type() ==
typeid(
txin_gen);
1252 for (
const auto &in: tx.
vin)
1253 {
1255 continue;
1256 const auto &txin = boost::get<txin_to_key>(in);
1257 if (opt_rct_only && txin.amount != 0)
1258 continue;
1259
1261 if (n == 0)
1263 add_key_image(txn,
output_data(txin.amount, out), txin.k_image);
1264
1265 std::vector<uint64_t> relative_ring;
1266 std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets);
1267 const uint32_t ring_size = txin.key_offsets.size();
1268 const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring);
1269 uint64_t pa_total = 0, pa_spent = 0;
1270 if (!opt_rct_only)
1271 get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent);
1272 if (n == 0 && ring_size == 1)
1273 {
1274 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]);
1275 if (opt_verbose)
1276 {
1277 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in a 1-ring");
1278 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1279 }
1280 blackballs.push_back(output);
1281 if (add_spent_output(cur,
output_data(txin.amount, absolute[0])))
1282 inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1");
1283 }
1284 else if (n == 0 && instances == new_ring.size())
1285 {
1286 for (size_t o = 0; o < new_ring.size(); ++o)
1287 {
1288 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1289 if (opt_verbose)
1290 {
1291 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in " << new_ring.size() <<
" identical " << new_ring.size() <<
"-rings");
1292 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1293 }
1294 blackballs.push_back(output);
1295 if (add_spent_output(cur,
output_data(txin.amount, absolute[o])))
1296 inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings");
1297 }
1298 }
1299 else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total)
1300 {
1301 for (size_t o = 0; o < pa_total; ++o)
1302 {
1303 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o);
1304 if (opt_verbose)
1305 {
1306 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to as many outputs of that amount being spent as exist so far");
1307 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1308 }
1309 blackballs.push_back(output);
1310 if (add_spent_output(cur,
output_data(txin.amount, o)))
1311 inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count");
1312 }
1313 }
1314 else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size())
1315 {
1316 for (size_t o = 0; o < new_ring.size(); ++o)
1317 {
1318 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
1319 if (opt_verbose)
1320 {
1321 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in " << new_ring.size() <<
" subsets of " << new_ring.size() <<
"-rings");
1322 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1323 }
1324 blackballs.push_back(output);
1325 if (add_spent_output(cur,
output_data(txin.amount, absolute[o])))
1326 inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings");
1327 }
1328 }
1329 else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring))
1330 {
1331 MDEBUG(
"Key image " << txin.k_image <<
" already seen: rings " <<
1332 boost::join(relative_ring | boost::adaptors::transformed([](
uint64_t out){return std::to_string(out);}),
" ") <<
1333 ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](
uint64_t out){return std::to_string(out);}),
" "));
1334 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1335 if (relative_ring != txin.key_offsets)
1336 {
1337 MDEBUG(
"Rings are different");
1338 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1341 std::vector<uint64_t> common;
1343 {
1344 if (std::find(r1.begin(), r1.end(), out) != r1.end())
1345 common.push_back(out);
1346 }
1347 if (common.empty())
1348 {
1349 MERROR(
"Rings for the same key image are disjoint");
1350 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1351 }
1352 else if (common.size() == 1)
1353 {
1354 const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]);
1355 if (opt_verbose)
1356 {
1357 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in rings with a single common element");
1358 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1359 }
1360 blackballs.push_back(output);
1361 if (add_spent_output(cur,
output_data(txin.amount, common[0])))
1362 inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack");
1363 }
1364 else
1365 {
1366 MDEBUG(
"The intersection has more than one element, it's still ok");
1367 std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
1368 for (const auto &out: r0)
1369 if (std::find(common.begin(), common.end(), out) != common.end())
1370 new_ring.push_back(out);
1372 }
1373 }
1374 }
1375 if (n == 0)
1376 {
1377 set_relative_ring(txn, txin.k_image, new_ring);
1378 if (!opt_rct_only)
1379 inc_per_amount_outputs(txn, txin.amount, 0, 1);
1380 }
1381 }
1382 set_processed_txidx(txn, canonical, start_idx+1);
1383 if (!opt_rct_only)
1384 {
1385 for (
const auto &out: tx.
vout)
1386 {
1388
1389 if (opt_rct_only && amount != 0)
1390 continue;
1392 continue;
1393 inc_per_amount_outputs(txn, amount, 1, 0);
1394 }
1395 }
1396
1397 ++records;
1398 if (records >= records_per_sync)
1399 {
1400 if (!blackballs.empty())
1401 {
1402 ringdb.blackball(blackballs);
1403 blackballs.clear();
1404 }
1408 int dbr = resize_env(cache_dir.c_str());
1414 records = 0;
1415 }
1416
1417 if (stop_requested)
1418 {
1419 MINFO(
"Stopping scan...");
1420 return false;
1421 }
1422 return true;
1423 });
1427 LOG_PRINT_L0(
"blockchain from " << inputs[n] <<
" processed till tx idx " << start_idx);
1428 if (stop_requested)
1429 break;
1430 }
1431
1432 std::vector<output_data> work_spent;
1433
1434 if (stop_requested)
1435 goto skip_secondary_passes;
1436
1437 if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs)
1438 {
1442 work_spent = get_spent_outputs(txn);
1444 }
1445
1446 while (!work_spent.empty())
1447 {
1448 LOG_PRINT_L0(
"Secondary pass on " << work_spent.size() <<
" spent outputs");
1449
1450 int dbr = resize_env(cache_dir.c_str());
1452
1459
1460 std::vector<std::pair<uint64_t, uint64_t>> blackballs;
1461 std::vector<output_data> scan_spent = std::move(work_spent);
1462 work_spent.clear();
1464 {
1465 std::vector<crypto::key_image> key_images = get_key_images(txn, od);
1467 {
1468 std::vector<uint64_t> relative_ring;
1471 size_t known = 0;
1474 {
1476 if (is_output_spent(cur, new_od))
1477 ++known;
1478 else
1480 }
1481 if (known == absolute.size() - 1)
1482 {
1483 const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown);
1484 if (opt_verbose)
1485 {
1486 MINFO(
"Marking output " << output.first <<
"/" << output.second <<
" as spent, due to being used in a " <<
1487 absolute.size() << "-ring where all other outputs are known to be spent");
1488 }
1489 blackballs.push_back(output);
1490 if (add_spent_output(cur,
output_data(od.amount, last_unknown)))
1491 inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction");
1492 work_spent.push_back(
output_data(od.amount, last_unknown));
1493 }
1494 }
1495
1496 if (stop_requested)
1497 {
1498 MINFO(
"Stopping secondary passes. Secondary passes are not incremental, they will re-run fully.");
1499 return 0;
1500 }
1501 }
1502 if (!blackballs.empty())
1503 {
1504 ringdb.blackball(blackballs);
1505 blackballs.clear();
1506 }
1510 }
1511
1512skip_secondary_passes:
1513 uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs;
1514 LOG_PRINT_L0(std::to_string(diff) <<
" new outputs marked as spent, " << get_num_spent_outputs() <<
" total outputs marked as spent");
1515
1520 get_num_outputs(txn0, cur0, dbi0, pre_rct,
rct);
1521 MINFO(
"Total pre-rct outputs: " << pre_rct);
1522 MINFO(
"Total rct outputs: " <<
rct);
1523 static const struct {
const char *
key;
uint64_t base; } stat_keys[] = {
1524 {
"pre-rct-ring-size-1", pre_rct }, {
"rct-ring-size-1",
rct },
1525 {
"pre-rct-duplicate-rings", pre_rct }, {
"rct-duplicate-rings",
rct },
1526 {
"pre-rct-subset-rings", pre_rct }, {
"rct-subset-rings",
rct },
1527 {
"pre-rct-full-count", pre_rct }, {
"rct-full-count",
rct },
1528 {
"pre-rct-key-image-attack", pre_rct }, {
"rct-key-image-attack",
rct },
1529 {
"pre-rct-extra", pre_rct }, {
"rct-ring-extra",
rct },
1530 {
"pre-rct-chain-reaction", pre_rct }, {
"rct-chain-reaction",
rct },
1531 };
1532 for (
const auto &
key: stat_keys)
1533 {
1535 if (!get_stat(txn,
key.key, data))
1536 data = 0;
1537 float percent =
key.base ? 100.0f * data /
key.base : 0.0f;
1538 MINFO(
key.key <<
": " << data <<
" (" << percent <<
"%)");
1539 }
1541
1542 if (!opt_export.empty())
1543 {
1550 export_spent_outputs(cur, opt_export);
1553 }
1554
1555 LOG_PRINT_L0(
"Blockchain spent output data exported OK");
1556 close_db(env0, txn0, cur0, dbi0);
1557 close();
1558 return 0;
1559
1561}
std::vector< txin_v > vin
std::vector< tx_out > vout
void mdb_txn_abort(MDB_txn *txn)
Abandon all the operations of the transaction instead of saving them.
int mdb_txn_commit(MDB_txn *txn)
Commit all the operations of a transaction into the database.
char * mdb_strerror(int err)
Return a string describing a given error code.
int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor)
Create a cursor handle.
void mdb_cursor_close(MDB_cursor *cursor)
Close a cursor handle.
int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn)
Create a transaction for use with the environment.
struct MDB_env MDB_env
Opaque structure for a database environment.
struct MDB_txn MDB_txn
Opaque structure for a transaction handle.
unsigned int MDB_dbi
A handle for an individual database in the DB environment.
struct MDB_cursor MDB_cursor
Opaque structure for navigating through a database.
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size=MAX_LOG_FILE_SIZE, const std::size_t max_log_files=MAX_LOG_FILES)
#define CATCH_ENTRY(location, return_val)
std::string mlog_get_default_log_path(const char *default_filename)
void mlog_set_log(const char *log)
#define CHECK_AND_ASSERT_THROW_MES(expr, message)
void add_arg(boost::program_options::options_description &description, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg, bool unique=true)
const arg_descriptor< bool > arg_help
bool is_arg_defaulted(const boost::program_options::variables_map &vm, const arg_descriptor< T, required, dependent, NUM_DEPS > &arg)
bool handle_error_helper(const boost::program_options::options_description &desc, F parser)
T get_arg(const boost::program_options::variables_map &vm, const arg_descriptor< T, false, true > &arg)
std::vector< uint64_t > absolute_output_offsets_to_relative(const std::vector< uint64_t > &off)
std::vector< uint64_t > relative_output_offsets_to_absolute(const std::vector< uint64_t > &off)
bool blockchain_valid_db_type(const std::string &db_type)
std::string blockchain_db_types(const std::string &sep)
const command_line::arg_descriptor< std::string > arg_db_sync_mode
const command_line::arg_descriptor< std::string > arg_log_level
const char *const ELECTRONEUM_RELEASE_NAME
const char *const ELECTRONEUM_VERSION_FULL