util/kvstore/LevelDBBenchmark.java

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 package org.apache.spark.util.kvstore;
0019
0020 import java.io.File;
0021 import java.util.ArrayList;
0022 import java.util.Collections;
0023 import java.util.List;
0024 import java.util.Map;
0025 import java.util.concurrent.atomic.AtomicInteger;
0026
0027 import com.codahale.metrics.MetricRegistry;
0028 import com.codahale.metrics.Slf4jReporter;
0029 import com.codahale.metrics.Snapshot;
0030 import com.codahale.metrics.Timer;
0031 import org.apache.commons.io.FileUtils;
0032 import org.junit.After;
0033 import org.junit.AfterClass;
0034 import org.junit.Before;
0035 import org.junit.Ignore;
0036 import org.junit.Test;
0037 import org.slf4j.LoggerFactory;
0038 import static org.junit.Assert.*;
0039
0040 /**
0041  * A set of small benchmarks for the LevelDB implementation.
0042  *
0043  * The benchmarks are run over two different types (one with just a natural index, and one
0044  * with a ref index), over a set of 2^20 elements, and the following tests are performed:
0045  *
0046  * - write (then update) elements in sequential natural key order
0047  * - write (then update) elements in random natural key order
0048  * - iterate over natural index, ascending and descending
0049  * - iterate over ref index, ascending and descending
0050  */
0051 @Ignore
0052 public class LevelDBBenchmark {
0053
0054   private static final int COUNT = 1024;
0055   private static final AtomicInteger IDGEN = new AtomicInteger();
0056   private static final MetricRegistry metrics = new MetricRegistry();
0057   private static final Timer dbCreation = metrics.timer("dbCreation");
0058   private static final Timer dbClose = metrics.timer("dbClose");
0059
0060   private LevelDB db;
0061   private File dbpath;
0062
0063   @Before
0064   public void setup() throws Exception {
0065     dbpath = File.createTempFile("test.", ".ldb");
0066     dbpath.delete();
0067     try(Timer.Context ctx = dbCreation.time()) {
0068       db = new LevelDB(dbpath);
0069     }
0070   }
0071
0072   @After
0073   public void cleanup() throws Exception {
0074     if (db != null) {
0075       try(Timer.Context ctx = dbClose.time()) {
0076         db.close();
0077       }
0078     }
0079     if (dbpath != null) {
0080       FileUtils.deleteQuietly(dbpath);
0081     }
0082   }
0083
0084   @AfterClass
0085   public static void report() {
0086     if (metrics.getTimers().isEmpty()) {
0087       return;
0088     }
0089
0090     int headingPrefix = 0;
0091     for (Map.Entry<String, Timer> e : metrics.getTimers().entrySet()) {
0092       headingPrefix = Math.max(e.getKey().length(), headingPrefix);
0093     }
0094     headingPrefix += 4;
0095
0096     StringBuilder heading = new StringBuilder();
0097     for (int i = 0; i < headingPrefix; i++) {
0098       heading.append(" ");
0099     }
0100     heading.append("\tcount");
0101     heading.append("\tmean");
0102     heading.append("\tmin");
0103     heading.append("\tmax");
0104     heading.append("\t95th");
0105     System.out.println(heading);
0106
0107     for (Map.Entry<String, Timer> e : metrics.getTimers().entrySet()) {
0108       StringBuilder row = new StringBuilder();
0109       row.append(e.getKey());
0110       for (int i = 0; i < headingPrefix - e.getKey().length(); i++) {
0111         row.append(" ");
0112       }
0113
0114       Snapshot s = e.getValue().getSnapshot();
0115       row.append("\t").append(e.getValue().getCount());
0116       row.append("\t").append(toMs(s.getMean()));
0117       row.append("\t").append(toMs(s.getMin()));
0118       row.append("\t").append(toMs(s.getMax()));
0119       row.append("\t").append(toMs(s.get95thPercentile()));
0120
0121       System.out.println(row);
0122     }
0123
0124     Slf4jReporter.forRegistry(metrics).outputTo(LoggerFactory.getLogger(LevelDBBenchmark.class))
0125       .build().report();
0126   }
0127
0128   private static String toMs(double nanos) {
0129     return String.format("%.3f", nanos / 1000 / 1000);
0130   }
0131
0132   @Test
0133   public void sequentialWritesNoIndex() throws Exception {
0134     List<SimpleType> entries = createSimpleType();
0135     writeAll(entries, "sequentialWritesNoIndex");
0136     writeAll(entries, "sequentialUpdatesNoIndex");
0137     deleteNoIndex(entries, "sequentialDeleteNoIndex");
0138   }
0139
0140   @Test
0141   public void randomWritesNoIndex() throws Exception {
0142     List<SimpleType> entries = createSimpleType();
0143
0144     Collections.shuffle(entries);
0145     writeAll(entries, "randomWritesNoIndex");
0146
0147     Collections.shuffle(entries);
0148     writeAll(entries, "randomUpdatesNoIndex");
0149
0150     Collections.shuffle(entries);
0151     deleteNoIndex(entries, "randomDeletesNoIndex");
0152   }
0153
0154   @Test
0155   public void sequentialWritesIndexedType() throws Exception {
0156     List<IndexedType> entries = createIndexedType();
0157     writeAll(entries, "sequentialWritesIndexed");
0158     writeAll(entries, "sequentialUpdatesIndexed");
0159     deleteIndexed(entries, "sequentialDeleteIndexed");
0160   }
0161
0162   @Test
0163   public void randomWritesIndexedTypeAndIteration() throws Exception {
0164     List<IndexedType> entries = createIndexedType();
0165
0166     Collections.shuffle(entries);
0167     writeAll(entries, "randomWritesIndexed");
0168
0169     Collections.shuffle(entries);
0170     writeAll(entries, "randomUpdatesIndexed");
0171
0172     // Run iteration benchmarks here since we've gone through the trouble of writing all
0173     // the data already.
0174     KVStoreView<?> view = db.view(IndexedType.class);
0175     iterate(view, "naturalIndex");
0176     iterate(view.reverse(), "naturalIndexDescending");
0177     iterate(view.index("name"), "refIndex");
0178     iterate(view.index("name").reverse(), "refIndexDescending");
0179
0180     Collections.shuffle(entries);
0181     deleteIndexed(entries, "randomDeleteIndexed");
0182   }
0183
0184   private void iterate(KVStoreView<?> view, String name) throws Exception {
0185     Timer create = metrics.timer(name + "CreateIterator");
0186     Timer iter = metrics.timer(name + "Iteration");
0187     KVStoreIterator<?> it = null;
0188     {
0189       // Create the iterator several times, just to have multiple data points.
0190       for (int i = 0; i < 1024; i++) {
0191         if (it != null) {
0192           it.close();
0193         }
0194         try(Timer.Context ctx = create.time()) {
0195           it = view.closeableIterator();
0196         }
0197       }
0198     }
0199
0200     for (; it.hasNext(); ) {
0201       try(Timer.Context ctx = iter.time()) {
0202         it.next();
0203       }
0204     }
0205   }
0206
0207   private void writeAll(List<?> entries, String timerName) throws Exception {
0208     Timer timer = newTimer(timerName);
0209     for (Object o : entries) {
0210       try(Timer.Context ctx = timer.time()) {
0211         db.write(o);
0212       }
0213     }
0214   }
0215
0216   private void deleteNoIndex(List<SimpleType> entries, String timerName) throws Exception {
0217     Timer delete = newTimer(timerName);
0218     for (SimpleType i : entries) {
0219       try(Timer.Context ctx = delete.time()) {
0220         db.delete(i.getClass(), i.key);
0221       }
0222     }
0223   }
0224
0225   private void deleteIndexed(List<IndexedType> entries, String timerName) throws Exception {
0226     Timer delete = newTimer(timerName);
0227     for (IndexedType i : entries) {
0228       try(Timer.Context ctx = delete.time()) {
0229         db.delete(i.getClass(), i.key);
0230       }
0231     }
0232   }
0233
0234   private List<SimpleType> createSimpleType() {
0235     List<SimpleType> entries = new ArrayList<>();
0236     for (int i = 0; i < COUNT; i++) {
0237       SimpleType t = new SimpleType();
0238       t.key = IDGEN.getAndIncrement();
0239       t.name = "name" + (t.key % 1024);
0240       entries.add(t);
0241     }
0242     return entries;
0243   }
0244
0245   private List<IndexedType> createIndexedType() {
0246     List<IndexedType> entries = new ArrayList<>();
0247     for (int i = 0; i < COUNT; i++) {
0248       IndexedType t = new IndexedType();
0249       t.key = IDGEN.getAndIncrement();
0250       t.name = "name" + (t.key % 1024);
0251       entries.add(t);
0252     }
0253     return entries;
0254   }
0255
0256   private Timer newTimer(String name) {
0257     assertNull("Timer already exists: " + name, metrics.getTimers().get(name));
0258     return metrics.timer(name);
0259   }
0260
0261   public static class SimpleType {
0262
0263     @KVIndex
0264     public int key;
0265
0266     public String name;
0267
0268   }
0269
0270   public static class IndexedType {
0271
0272     @KVIndex
0273     public int key;
0274
0275     @KVIndex("name")
0276     public String name;
0277
0278   }
0279
0280 }