Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[core] Support record count and opcount exceeding 2^31 #1742

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+20 −20
Open

[core] Support record count and opcount exceeding 2^31 #1742

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions core/src/main/java/site/ycsb/Client.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ public static boolean checkRequiredProperties(Properties props) {
*
* @throws IOException Either failed to write to output stream or failed to close it.
*/
private static void exportMeasurements(Properties props, int opcount, long runtime)
private static void exportMeasurements(Properties props, long opcount, long runtime)
throws IOException {
MeasurementsExporter exporter = null;
try {
Expand Down Expand Up @@ -408,18 +408,18 @@ private static List<ClientThread> initDb(String dbname, Properties props, int th

final List<ClientThread> clients = new ArrayList<>(threadcount);
try (final TraceScope span = tracer.newScope(CLIENT_INIT_SPAN)) {
int opcount;
long opcount;
if (dotransactions) {
opcount = Integer.parseInt(props.getProperty(OPERATION_COUNT_PROPERTY, "0"));
opcount = Long.parseLong(props.getProperty(OPERATION_COUNT_PROPERTY, "0"));
} else {
if (props.containsKey(INSERT_COUNT_PROPERTY)) {
opcount = Integer.parseInt(props.getProperty(INSERT_COUNT_PROPERTY, "0"));
opcount = Long.parseLong(props.getProperty(INSERT_COUNT_PROPERTY, "0"));
} else {
opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT));
opcount = Long.parseLong(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT));
}
}
if (threadcount > opcount && opcount > 0){
threadcount = opcount;
threadcount = (int)opcount;
System.out.println("Warning: the threadcount is bigger than recordcount, the threadcount will be recordcount!");
}
for (int threadid = 0; threadid < threadcount; threadid++) {
Expand All @@ -432,7 +432,7 @@ private static List<ClientThread> initDb(String dbname, Properties props, int th
break;
}

int threadopcount = opcount / threadcount;
long threadopcount = opcount / threadcount;

// ensure correct number of operations, in case opcount is not a multiple of threadcount
if (threadid < opcount % threadcount) {
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/java/site/ycsb/ClientThread.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ public class ClientThread implements Runnable {
private DB db;
private boolean dotransactions;
private Workload workload;
private int opcount;
private long opcount;
private double targetOpsPerMs;

private int opsdone;
private long opsdone;
private int threadid;
private int threadcount;
private Object workloadstate;
Expand All @@ -56,7 +56,7 @@ public class ClientThread implements Runnable {
* @param targetperthreadperms target number of operations per thread per ms
* @param completeLatch The latch tracking the completion of all clients.
*/
public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, int opcount,
public ClientThread(DB db, boolean dotransactions, Workload workload, Properties props, long opcount,
double targetperthreadperms, CountDownLatch completeLatch) {
this.db = db;
this.dotransactions = dotransactions;
Expand All @@ -81,7 +81,7 @@ public void setThreadCount(final int threadCount) {
threadcount = threadCount;
}

public int getOpsDone() {
public long getOpsDone() {
return opsdone;
}

Expand Down Expand Up @@ -179,8 +179,8 @@ private void throttleNanos(long startTimeNanos) {
/**
* The total amount of work this thread is still expected to do.
*/
int getOpsTodo() {
int todo = opcount - opsdone;
long getOpsTodo() {
long todo = opcount - opsdone;
return todo < 0 ? 0 : todo;
}
}
2 changes: 1 addition & 1 deletion core/src/main/java/site/ycsb/generator/ScrambledZipfianGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
public class ScrambledZipfianGenerator extends NumberGenerator {
public static final double ZETAN = 26.46902820178302;
public static final double USED_ZIPFIAN_CONSTANT = 0.99;
public static final long ITEM_COUNT = 10000000000L;
public static final long ITEM_COUNT = 1000000000000L;

private ZipfianGenerator gen;
private final long min, max, itemcount;
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/java/site/ycsb/workloads/CoreWorkload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ public void init(Properties p) throws WorkloadException {
recordcount =
Long.parseLong(p.getProperty(Client.RECORD_COUNT_PROPERTY, Client.DEFAULT_RECORD_COUNT));
if (recordcount == 0) {
recordcount = Integer.MAX_VALUE;
recordcount = Long.MAX_VALUE;
}
String requestdistrib =
p.getProperty(REQUEST_DISTRIBUTION_PROPERTY, REQUEST_DISTRIBUTION_PROPERTY_DEFAULT);
Expand All @@ -448,7 +448,7 @@ public void init(Properties p) throws WorkloadException {
long insertstart =
Long.parseLong(p.getProperty(INSERT_START_PROPERTY, INSERT_START_PROPERTY_DEFAULT));
long insertcount=
Integer.parseInt(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart)));
Long.parseLong(p.getProperty(INSERT_COUNT_PROPERTY, String.valueOf(recordcount - insertstart)));
// Confirm valid values for insertstart and insertcount in relation to recordcount
if (recordcount < (insertstart + insertcount)) {
System.err.println("Invalid combination of insertstart, insertcount and recordcount.");
Expand Down Expand Up @@ -513,8 +513,8 @@ public void init(Properties p) throws WorkloadException {
// the keyspace doesn't change from the perspective of the scrambled zipfian generator
final double insertproportion = Double.parseDouble(
p.getProperty(INSERT_PROPORTION_PROPERTY, INSERT_PROPORTION_PROPERTY_DEFAULT));
int opcount = Integer.parseInt(p.getProperty(Client.OPERATION_COUNT_PROPERTY));
int expectednewkeys = (int) ((opcount) * insertproportion * 2.0); // 2 is fudge factor
long opcount = Long.parseLong(p.getProperty(Client.OPERATION_COUNT_PROPERTY));
long expectednewkeys = (long) ((opcount) * insertproportion * 2.0); // 2 is fudge factor

keychooser = new ScrambledZipfianGenerator(insertstart, insertstart + insertcount + expectednewkeys);
} else if (requestdistrib.compareTo("latest") == 0) {
Expand Down Expand Up @@ -611,7 +611,7 @@ private String buildDeterministicValue(String key, String fieldkey) {
*/
@Override
public boolean doInsert(DB db, Object threadstate) {
int keynum = keysequence.nextValue().intValue();
long keynum = keysequence.nextValue().longValue();
String dbkey = CoreWorkload.buildKeyName(keynum, zeropadding, orderedinserts);
HashMap<String, ByteIterator> values = buildValues(dbkey);

Expand Down Expand Up @@ -709,7 +709,7 @@ long nextKeynum() {
long keynum;
if (keychooser instanceof ExponentialGenerator) {
do {
keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().intValue();
keynum = transactioninsertkeysequence.lastValue() - keychooser.nextValue().longValue();
} while (keynum < 0);
} else {
do {
Expand Down