diff --git a/tools/toolkit/README.md b/tools/toolkit/README.md index 082fb589..ffce129b 100644 --- a/tools/toolkit/README.md +++ b/tools/toolkit/README.md @@ -210,11 +210,26 @@ reward = [ "TLyqzVGLV1srkB7dToTAEqgDSfPtXRJZYH", "TJvaAeFb8Lykt9RQcVyyTFN2iDvGMuyD4M" ] + +voters = { + allWitnesses = false + allActiveWitnesses = true + witnessList = [ + "TQopP5GM68QoqLzpz8YReDfSoCMkvwcZYd", + "TDpt9adA6QidL1B1sy3D8NC717C6L5JxFo", + ] + threshold = 10000 +} ``` - `vote.allWitnesses`: configure whether to query the vote information of all witnesses. - `vote.witnessList`: configure to query the vote information from specified witness list. The option is valid only when `vote.allWitnesses = false`. - `reward`: configure the address list you need to query the reward. +- `voters.allWitnesses`: configure whether to query the voter information of all witnesses. +- `voters.allActiveWitnesses`: configure whether to query the voter information of all active + witnesses. +- `voters.witnessList`: configure to query the voter information from specified witness list. +- `voters.threshold`: configure the votes count threshold to filter the voters. Execute query command. ```shell script diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java b/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java index 16dc2ff3..279176a1 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java +++ b/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java @@ -14,9 +14,14 @@ import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; import static org.tron.plugins.utils.Constant.NEW_REWARD_ALGORITHM_EFFECTIVE_CYCLE; import static org.tron.plugins.utils.Constant.REWARDS_KEY; +import static org.tron.plugins.utils.Constant.VOTERS_ALL_ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.VOTERS_ALL_WITNESSES; +import static org.tron.plugins.utils.Constant.VOTERS_WITNESS_LIST; +import static org.tron.plugins.utils.Constant.VOTERS_WITNESS_THRESHOLD; import static org.tron.plugins.utils.Constant.VOTES_ALL_WITNESSES; import static org.tron.plugins.utils.Constant.VOTES_STORE; import static org.tron.plugins.utils.Constant.VOTES_WITNESS_LIST; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; import static org.tron.plugins.utils.Constant.WITNESS_STORE; import com.google.common.collect.Maps; @@ -29,6 +34,7 @@ import java.math.BigInteger; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -54,6 +60,7 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.exception.BadItemException; import org.tron.core.store.DelegationStore; +import org.tron.plugins.utils.Constant; import org.tron.plugins.utils.FileUtils; import org.tron.plugins.utils.JsonFormat; import org.tron.plugins.utils.db.DBInterface; @@ -74,7 +81,7 @@ public class DbQuery implements Callable { @CommandLine.Spec - CommandLine.Model.CommandSpec spec; + static CommandLine.Model.CommandSpec spec; @CommandLine.Option(names = {"-d", "--database-directory"}, defaultValue = "output-directory", @@ -97,6 +104,7 @@ public class DbQuery implements Callable { private DBInterface blockStore; private DBInterface accountStore; private DBInterface delegationStore; + private DBInterface witnessScheduleStore; boolean allWitness = false; List witnessList = new ArrayList<>(); @@ -106,6 +114,11 @@ public class DbQuery implements Callable { List rewardAddressList = new ArrayList<>(); Map latestWitnessVi = new HashMap<>(); + boolean allWitnessVoters = false; + boolean allActiveWitnesses = false; + List votersWitnessList = new ArrayList<>(); + long threshold = 0; + private void initStore() throws IOException, RocksDBException { String srcDir = database + File.separator + "database"; witnessStore = DbTool.getDB(srcDir, WITNESS_STORE); @@ -115,6 +128,7 @@ private void initStore() throws IOException, RocksDBException { blockStore = DbTool.getDB(srcDir, BLOCK_STORE); accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); delegationStore = DbTool.getDB(srcDir, DELEGATION_STORE); + witnessScheduleStore = DbTool.getDB(srcDir, WITNESS_SCHEDULE_STORE); } @@ -149,6 +163,7 @@ public Integer call() throws Exception { initStore(); processVotes(queryConfig); processRewards(queryConfig); + processVoters(queryConfig); DbTool.close(); return 0; @@ -283,6 +298,12 @@ private void processVotes(Config queryConfig) throws BadItemException { spec.commandLine().getOut().println(cnt.get() + " " + output); logger.info(cnt.get() + " " + output); }); + + try { + iterator.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } } private String formatWitness(WitnessCapsule witnessCapsule, long previousCnt) { @@ -413,6 +434,168 @@ private Boolean existInWitnessList(VotesCapsule votesCapsule, List witne return exist.get(); } + private void processVoters(Config config) { + if (config.hasPath(VOTERS_ALL_WITNESSES)) { + allWitnessVoters = config.getBoolean(VOTERS_ALL_WITNESSES); + } + if (config.hasPath(VOTERS_ALL_ACTIVE_WITNESSES)) { + allActiveWitnesses = config.getBoolean(VOTERS_ALL_ACTIVE_WITNESSES); + } + if (config.hasPath(VOTERS_WITNESS_LIST)) { + votersWitnessList = config.getStringList(VOTERS_WITNESS_LIST); + } + + if (!allWitnessVoters && !allActiveWitnesses && votersWitnessList.isEmpty()) { + spec.commandLine().getOut() + .println("skip the voters query."); + logger.info("skip the voters query."); + return; + } + + if (config.hasPath(VOTERS_WITNESS_THRESHOLD)) { + threshold = config.getLong(VOTERS_WITNESS_THRESHOLD); + } + spec.commandLine().getOut() + .println("start voters query."); + logger.info("start the voters query."); + long start = System.currentTimeMillis(); + Map> votersMap = new HashMap<>(); + DBIterator iterator = accountStore.iterator(); + spec.commandLine().getOut().format("start to query account store: ").println(); + logger.info("begin to query account store: "); + AccountCapsule accountCapsule; + long accountCnt = 0; + for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { + accountCapsule = new AccountCapsule(iterator.getValue()); + ByteString owner = accountCapsule.getAddress(); + accountCapsule.getVotesList().forEach(vote -> { + votersMap.computeIfAbsent(vote.getVoteAddress(), k -> new HashMap<>()); + votersMap.get(vote.getVoteAddress()).put(owner, vote.getVoteCount()); + }); + + accountCnt++; + if (accountCnt % 10000000 == 0) { + spec.commandLine().getOut().format("processed %d accounts", accountCnt) + .println(); + logger.info("processed {} accounts", accountCnt); + } + } + + long end = System.currentTimeMillis(); + spec.commandLine().getOut() + .format("account db query finished, total time: %d ms.", end - start).println(); + logger.info("account db query finished, total time: {} ms.", end - start); + + Map witnessesMap = new HashMap<>(); + Map oldWitnessCnt = new HashMap<>(); + DBIterator witnessIterator = witnessStore.iterator(); + WitnessCapsule witnessCapsule; + for (witnessIterator.seekToFirst(); witnessIterator.valid(); witnessIterator.next()) { + witnessCapsule = new WitnessCapsule(witnessIterator.getValue()); + witnessesMap.put(ByteString.copyFrom(witnessIterator.getKey()), witnessCapsule); + oldWitnessCnt.put(witnessCapsule.getAddress(), witnessCapsule.getVoteCount()); + } + Map countWitness = countVote(); + countWitness.forEach((address, voteCount) -> { + WitnessCapsule witness = witnessesMap.get(address); + if (witness == null) { + throw new IllegalStateException("witness not exist"); + } + witness.setVoteCount(witness.getVoteCount() + voteCount); + witnessesMap.put(address, witness); + }); + + AtomicInteger cnt = new AtomicInteger(); + cnt.set(-1); + if (allWitnessVoters) { + votersMap.entrySet().stream() + .forEach(entry -> { + WitnessCapsule witnessTmp = witnessesMap.get(entry.getKey()); + if (witnessTmp == null) { + throw new IllegalStateException("witness not exist"); + } + cnt.getAndIncrement(); + String output = formatWitness(witnessTmp, oldWitnessCnt.get(entry.getKey())); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + String witness = StringUtil.encode58Check(entry.getKey().toByteArray()); + spec.commandLine().getOut() + .format("List the voters of witness: %s with votes great than %d: %s.", witness, + threshold).println(); + printVoters(entry.getValue(), threshold); + }); + } else if (allActiveWitnesses) { + byte[] witnesses = witnessScheduleStore.get(Constant.ACTIVE_WITNESSES); + List activeWitnesses = decodeActiveWitness(witnesses); + spec.commandLine().getOut().format("There are %d active witnesses in db", + activeWitnesses.size()).println(); + logger.info("There are {} active witnesses in db", activeWitnesses.size()); + activeWitnesses.forEach(witness -> { + WitnessCapsule witnessTmp = witnessesMap.get(witness); + if (witnessTmp == null) { + throw new IllegalStateException("witness not exist"); + } + cnt.getAndIncrement(); + String output = formatWitness(witnessTmp, oldWitnessCnt.get(witness)); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + String witnessBase58 = StringUtil.encode58Check(witness.toByteArray()); + spec.commandLine().getOut() + .format("List the voters of active witness: %s with votes great than %d.", + witnessBase58, threshold).println(); + logger.info("List the voters of active witness: {} with votes great than {}.", + witnessBase58, threshold); + printVoters(votersMap.get(witness), threshold); + }); + } else { + votersWitnessList.forEach(witness -> { + ByteString address = ByteString.copyFrom(Commons.decode58Check(witness)); + if (!witnessesMap.containsKey(address)) { + spec.commandLine().getErr().format("address: %s is not a witness", witness).println(); + logger.error("address: {} is not a witness", witness); + return; + } + + cnt.getAndIncrement(); + String output = formatWitness(witnessesMap.get(address), oldWitnessCnt.get(address)); + spec.commandLine().getOut().println(cnt.get() + " " + output); + + spec.commandLine().getOut() + .format("List the voters of witness: %s with votes great than %d.", witness, + threshold).println(); + logger.info("List the voters of witness: {} with votes great than {}.", witness, threshold); + Map votesMap = votersMap.get(address); + printVoters(votesMap, threshold); + }); + } + + spec.commandLine().getOut().println("End of voters query."); + logger.info("End of voters query."); + try { + iterator.close(); + witnessIterator.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void printVoters(Map votersMap, long threshold) { + if (votersMap == null || votersMap.isEmpty()) { + return; + } + votersMap.entrySet().stream() + .filter(entry -> entry.getValue() >= threshold) + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .forEach(entry -> { + String witness = StringUtil.encode58Check(entry.getKey().toByteArray()); + long voteCount = entry.getValue(); + spec.commandLine().getOut() + .format("voterAddress: %s, vote count: %d", witness, voteCount).println(); + logger.info("voterAddress: {}, vote count: {}", witness, voteCount); + }); + } + + private void processRewards(Config config) { if (config.hasPath(REWARDS_KEY)) { rewardAddressList = config.getStringList(REWARDS_KEY); @@ -699,4 +882,18 @@ private long getOldReward(long begin, long end, List> votes) return reward; } + public static List decodeActiveWitness(byte[] witnesses) { + if (ByteArray.isEmpty(witnesses) + || witnesses.length % Constant.ADDRESS_BYTE_ARRAY_LENGTH != 0) { + throw new IllegalArgumentException("witnesses is invalid"); + } + + List witnessList = new ArrayList<>(); + for (int i = 0; i < witnesses.length; i += Constant.ADDRESS_BYTE_ARRAY_LENGTH) { + witnessList.add(ByteString.copyFrom( + Arrays.copyOfRange(witnesses, i, i + Constant.ADDRESS_BYTE_ARRAY_LENGTH))); + } + return witnessList; + } + } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java index 7fcc380c..ca9f4a36 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java +++ b/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java @@ -53,6 +53,13 @@ public class Constant { public static final String VOTES_WITNESS_LIST = "vote.witnessList"; public static final int BLOCK_PRODUCED_INTERVAL = 3000; + public static final String VOTERS_ALL_WITNESSES = "voters.allWitnesses"; + public static final String VOTERS_ALL_ACTIVE_WITNESSES = "voters.allActiveWitnesses"; + + public static final String VOTERS_WITNESS_LIST = "voters.witnessList"; + public static final String VOTERS_WITNESS_THRESHOLD = "voters.threshold"; + + public static final String REWARDS_KEY = "reward"; public static final byte[] CHANGE_DELEGATION = "CHANGE_DELEGATION".getBytes(); public static final byte[] CURRENT_CYCLE_NUMBER = "CURRENT_CYCLE_NUMBER".getBytes(); diff --git a/tools/toolkit/src/main/resources/query.conf b/tools/toolkit/src/main/resources/query.conf index a1df2aee..4fcdb4cd 100644 --- a/tools/toolkit/src/main/resources/query.conf +++ b/tools/toolkit/src/main/resources/query.conf @@ -18,3 +18,13 @@ reward = [ "TDaQAfV7gZSPzx5FZ23kenPwLjr3AGxnmH", "TXwrVcWA2BPRMs9gGz5DciT6iwdDKagbpB" ] + +voters = { + allWitnesses = false + allActiveWitnesses = true + witnessList = [ + "TMEQ4hu7DtLTHPn6qPsYktk1QGdTbAyTEK", + "THKJYuUmMKKARNf7s2VT51g5uPY6KEqnat" + ] + threshold = 10000000 +}