From fd9559688c1e667dd1891f39f29d467f2c754b62 Mon Sep 17 00:00:00 2001 From: TAKAMI Torao Date: Wed, 18 Dec 2024 21:37:27 +0900 Subject: [PATCH] allow loading of records exceeding 2^31 --- core/src/main/java/site/ycsb/Client.java | 14 +++++++------- core/src/main/java/site/ycsb/ClientThread.java | 12 ++++++------ .../ycsb/generator/ScrambledZipfianGenerator.java | 2 +- .../java/site/ycsb/workloads/CoreWorkload.java | 12 ++++++------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/site/ycsb/Client.java b/core/src/main/java/site/ycsb/Client.java index bd96821170..5fb2ccabc0 100644 --- a/core/src/main/java/site/ycsb/Client.java +++ b/core/src/main/java/site/ycsb/Client.java @@ -211,7 +211,7 @@ public static boolean checkRequiredProperties(Properties props) { * * @throws IOException Either failed to write to output stream or failed to close it. */ - private static void exportMeasurements(Properties props, int opcount, long runtime) + private static void exportMeasurements(Properties props, long opcount, long runtime) throws IOException { MeasurementsExporter exporter = null; try { @@ -408,18 +408,18 @@ private static List initDb(String dbname, Properties props, int th final List clients = new ArrayList<>(threadcount); try (final TraceScope span = tracer.newScope(CLIENT_INIT_SPAN)) { - int opcount; + long opcount; if (dotransactions) { - opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0")); + opcount = Long.parseLong(props.getProperty(OPERATION_COUNT_PROPERTY, "0")); } else { if (props.containsKey(INSERT_COUNT_PROPERTY)) { - opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0")); + opcount = Long.parseLong(props.getProperty(INSERT_COUNT_PROPERTY, "0")); } else { - opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT)); + opcount = Long.parseLong(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT)); } } if (threadcount > opcount && opcount > 0){ - threadcount = opcount; + threadcount = (int)opcount; System.out.println("Warning: the threadcount is bigger than recordcount, the threadcount will be recordcount!"); } for (int threadid = 0; threadid < threadcount; threadid++) { @@ -432,7 +432,7 @@ private static List initDb(String dbname, Properties props, int th break; } - int threadopcount = opcount / threadcount; + long threadopcount = opcount / threadcount; // ensure correct number of operations, in case opcount is not a multiple of threadcount if (threadid < opcount % threadcount) { diff --git a/core/src/main/java/site/ycsb/ClientThread.java b/core/src/main/java/site/ycsb/ClientThread.java index fcfc13267d..f75571f541 100644 --- a/core/src/main/java/site/ycsb/ClientThread.java +++ b/core/src/main/java/site/ycsb/ClientThread.java @@ -34,10 +34,10 @@ public class ClientThread implements Runnable { private DB db; private boolean dotransactions; private Workload workload; - private int opcount; + private long opcount; private double targetOpsPerMs; - private int opsdone; + private long opsdone; private int threadid; private int threadcount; private Object workloadstate; @@ -56,7 +56,7 @@ public class ClientThread implements Runnable { * @param targetperthreadperms target number of operations per thread per ms * @param completeLatch The latch tracking the completion of all clients. */ - public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, int opcount, + public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, long opcount, double targetperthreadperms, CountDownLatch completeLatch) { this.db = db; this.dotransactions = dotransactions; @@ -81,7 +81,7 @@ public void setThreadCount(final int threadCount) { threadcount = threadCount; } - public int getOpsDone() { + public long getOpsDone() { return opsdone; } @@ -179,8 +179,8 @@ private void throttleNanos(long startTimeNanos) { /** * The total amount of work this thread is still expected to do. */ - int getOpsTodo() { - int todo = opcount - opsdone; + long getOpsTodo() { + long todo = opcount - opsdone; return todo < 0 ? 0 : todo; } } diff --git a/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java b/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java index 155a6cf1dc..f7c18aa33c 100644 --- a/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java +++ b/core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java @@ -32,7 +32,7 @@ public class ScrambledZipfianGenerator extends NumberGenerator { public static final double ZETAN = 26.46902820178302; public static final double USED_ZIPFIAN_CONSTANT = 0.99; - public static final long ITEM_COUNT = 10000000000L; + public static final long ITEM_COUNT = 1000000000000L; private ZipfianGenerator gen; private final long min, max, itemcount; diff --git a/core/src/main/java/site/ycsb/workloads/CoreWorkload.java b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java index 082cb0dc2b..5734344502 100644 --- a/core/src/main/java/site/ycsb/workloads/CoreWorkload.java +++ b/core/src/main/java/site/ycsb/workloads/CoreWorkload.java @@ -434,7 +434,7 @@ public void init(Properties p) throws WorkloadException { recordcount = Long.parseLong(p.getProperty(Client.RECORD_COUNT_PROPERTY, Client.DEFAULT_RECORD_COUNT)); if (recordcount == 0) { - recordcount = Integer.MAX_VALUE; + recordcount = Long.MAX_VALUE; } String requestdistrib = p.getProperty(REQUEST_DISTRIBUTION_PROPERTY, REQUEST_DISTRIBUTION_PROPERTY_DEFAULT); @@ -448,7 +448,7 @@ public void init(Properties p) throws WorkloadException { long insertstart = Long.parseLong(p.getProperty(INSERT_START_PROPERTY, INSERT_START_PROPERTY_DEFAULT)); long insertcount= - Integer.parseInt(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart))); + Long.parseLong(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart))); // Confirm valid values for insertstart and insertcount in relation to recordcount if (recordcount < (insertstart + insertcount)) { System.err.println("Invalid combination of insertstart, insertcount and recordcount."); @@ -513,8 +513,8 @@ public void init(Properties p) throws WorkloadException { // the keyspace doesn't change from the perspective of the scrambled zipfian generator final double insertproportion = Double.parseDouble( p.getProperty(INSERT_PROPORTION_PROPERTY, INSERT_PROPORTION_PROPERTY_DEFAULT)); - int opcount = Integer.parseInt(p.getProperty(Client.OPERATION_COUNT_PROPERTY)); - int expectednewkeys = (int) ((opcount) * insertproportion * 2.0); // 2 is fudge factor + long opcount = Long.parseLong(p.getProperty(Client.OPERATION_COUNT_PROPERTY)); + long expectednewkeys = (long) ((opcount) * insertproportion * 2.0); // 2 is fudge factor keychooser = new ScrambledZipfianGenerator(insertstart, insertstart + insertcount + expectednewkeys); } else if (requestdistrib.compareTo("latest") == 0) { @@ -611,7 +611,7 @@ private String buildDeterministicValue(String key, String fieldkey) { */ @Override public boolean doInsert(DB db, Object threadstate) { - int keynum = keysequence.nextValue().intValue(); + long keynum = keysequence.nextValue().longValue(); String dbkey = CoreWorkload.buildKeyName(keynum, zeropadding, orderedinserts); HashMap values = buildValues(dbkey); @@ -709,7 +709,7 @@ long nextKeynum() { long keynum; if (keychooser instanceof ExponentialGenerator) { do { - keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().intValue(); + keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().longValue(); } while (keynum < 0); } else { do {