001/* 002 * Copyright (C) 2011 The Guava Authors 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 005 * in compliance with the License. You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software distributed under the License 010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 011 * or implied. See the License for the specific language governing permissions and limitations under 012 * the License. 013 */ 014 015package com.google.common.hash; 016 017import static com.google.common.base.Preconditions.checkArgument; 018import static com.google.common.base.Preconditions.checkNotNull; 019 020import com.google.common.annotations.J2ktIncompatible; 021import com.google.errorprone.annotations.Immutable; 022import com.google.j2objc.annotations.J2ObjCIncompatible; 023import java.security.Key; 024import java.util.ArrayList; 025import java.util.Arrays; 026import java.util.Collections; 027import java.util.Iterator; 028import java.util.List; 029import java.util.zip.Adler32; 030import java.util.zip.CRC32; 031import java.util.zip.CRC32C; 032import java.util.zip.Checksum; 033import javax.crypto.spec.SecretKeySpec; 034import org.jspecify.annotations.Nullable; 035 036/** 037 * Static methods to obtain {@link HashFunction} instances, and other static hashing-related 038 * utilities. 039 * 040 * <p>A comparison of the various hash functions can be found <a 041 * href="https://docs.google.com/spreadsheets/d/1_q2EVcxA2HjcrlVMbaqXwMj31h9M5-Bqj_m8vITOwwk/">here</a>. 042 * 043 * @author Kevin Bourrillion 044 * @author Dimitris Andreou 045 * @author Kurt Alfred Kluever 046 * @since 11.0 047 */ 048public final class Hashing { 049 /** 050 * Returns a general-purpose, <b>temporary-use</b>, non-cryptographic hash function. The algorithm 051 * the returned function implements is unspecified and subject to change without notice. 052 * 053 * <p><b>Warning:</b> a new random seed for these functions is chosen each time the {@code 054 * Hashing} class is loaded. <b>Do not use this method</b> if hash codes may escape the current 055 * process in any way, for example being sent over RPC, or saved to disk. For a general-purpose, 056 * non-cryptographic hash function that will never change behavior, we suggest {@link 057 * #murmur3_128}. 058 * 059 * <p>Repeated calls to this method on the same loaded {@code Hashing} class, using the same value 060 * for {@code minimumBits}, will return identically-behaving {@link HashFunction} instances. 061 * 062 * @param minimumBits a positive integer. This can be arbitrarily large. The returned {@link 063 * HashFunction} instance may use memory proportional to this integer. 064 * @return a hash function, described above, that produces hash codes of length {@code 065 * minimumBits} or greater 066 */ 067 @J2ktIncompatible 068 public static HashFunction goodFastHash(int minimumBits) { 069 int bits = checkPositiveAndMakeMultipleOf32(minimumBits); 070 071 if (bits == 32) { 072 return Murmur3_32HashFunction.GOOD_FAST_HASH_32; 073 } 074 if (bits <= 128) { 075 return Murmur3_128HashFunction.GOOD_FAST_HASH_128; 076 } 077 078 // Otherwise, join together some 128-bit murmur3s 079 int hashFunctionsNeeded = (bits + 127) / 128; 080 HashFunction[] hashFunctions = new HashFunction[hashFunctionsNeeded]; 081 hashFunctions[0] = Murmur3_128HashFunction.GOOD_FAST_HASH_128; 082 int seed = GOOD_FAST_HASH_SEED; 083 for (int i = 1; i < hashFunctionsNeeded; i++) { 084 seed += 1500450271; // a prime; shouldn't matter 085 hashFunctions[i] = murmur3_128(seed); 086 } 087 return new ConcatenatedHashFunction(hashFunctions); 088 } 089 090 /** 091 * Used to randomize {@link #goodFastHash} instances, so that programs which persist anything 092 * dependent on the hash codes they produce will fail sooner. 093 */ 094 @SuppressWarnings("GoodTime") // reading system time without TimeSource 095 static final int GOOD_FAST_HASH_SEED = (int) System.currentTimeMillis(); 096 097 /** 098 * Returns a hash function implementing the <a 099 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">32-bit murmur3 100 * algorithm, x86 variant</a> (little-endian variant), using the given seed value, <b>with a known 101 * bug</b> as described in the deprecation text. 102 * 103 * <p>The C++ equivalent is the MurmurHash3_x86_32 function (Murmur3A), which however does not 104 * have the bug. 105 * 106 * @deprecated This implementation produces incorrect hash values from the {@link 107 * HashFunction#hashString} method if the string contains non-BMP characters. Use {@link 108 * #murmur3_32_fixed(int)} instead. 109 */ 110 @Deprecated 111 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 112 @J2ktIncompatible 113 public static HashFunction murmur3_32(int seed) { 114 return new Murmur3_32HashFunction(seed, /* supplementaryPlaneFix= */ false); 115 } 116 117 /** 118 * Returns a hash function implementing the <a 119 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">32-bit murmur3 120 * algorithm, x86 variant</a> (little-endian variant), using the given seed value, <b>with a known 121 * bug</b> as described in the deprecation text. 122 * 123 * <p>The C++ equivalent is the MurmurHash3_x86_32 function (Murmur3A), which however does not 124 * have the bug. 125 * 126 * @deprecated This implementation produces incorrect hash values from the {@link 127 * HashFunction#hashString} method if the string contains non-BMP characters. Use {@link 128 * #murmur3_32_fixed()} instead. 129 */ 130 @Deprecated 131 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 132 @J2ktIncompatible 133 public static HashFunction murmur3_32() { 134 return Murmur3_32HashFunction.MURMUR3_32; 135 } 136 137 /** 138 * Returns a hash function implementing the <a 139 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">32-bit murmur3 140 * algorithm, x86 variant</a> (little-endian variant), using the given seed value. 141 * 142 * <p>The exact C++ equivalent is the MurmurHash3_x86_32 function (Murmur3A). 143 * 144 * <p>This method is called {@code murmur3_32_fixed} because it fixes a bug in the {@code 145 * HashFunction} returned by the original {@code murmur3_32} method. 146 * 147 * @since 31.0 148 */ 149 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 150 @J2ktIncompatible 151 public static HashFunction murmur3_32_fixed(int seed) { 152 return new Murmur3_32HashFunction(seed, /* supplementaryPlaneFix= */ true); 153 } 154 155 /** 156 * Returns a hash function implementing the <a 157 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">32-bit murmur3 158 * algorithm, x86 variant</a> (little-endian variant), using a seed value of zero. 159 * 160 * <p>The exact C++ equivalent is the MurmurHash3_x86_32 function (Murmur3A). 161 * 162 * <p>This method is called {@code murmur3_32_fixed} because it fixes a bug in the {@code 163 * HashFunction} returned by the original {@code murmur3_32} method. 164 * 165 * @since 31.0 166 */ 167 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 168 @J2ktIncompatible 169 public static HashFunction murmur3_32_fixed() { 170 return Murmur3_32HashFunction.MURMUR3_32_FIXED; 171 } 172 173 /** 174 * Returns a hash function implementing the <a 175 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">128-bit murmur3 176 * algorithm, x64 variant</a> (little-endian variant), using the given seed value. 177 * 178 * <p>The exact C++ equivalent is the MurmurHash3_x64_128 function (Murmur3F). 179 */ 180 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 181 @J2ktIncompatible 182 public static HashFunction murmur3_128(int seed) { 183 return new Murmur3_128HashFunction(seed); 184 } 185 186 /** 187 * Returns a hash function implementing the <a 188 * href="https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp">128-bit murmur3 189 * algorithm, x64 variant</a> (little-endian variant), using a seed value of zero. 190 * 191 * <p>The exact C++ equivalent is the MurmurHash3_x64_128 function (Murmur3F). 192 */ 193 @SuppressWarnings("IdentifierName") // the best we could do for adjacent digit blocks 194 @J2ktIncompatible 195 public static HashFunction murmur3_128() { 196 return Murmur3_128HashFunction.MURMUR3_128; 197 } 198 199 /** 200 * Returns a hash function implementing the <a href="https://131002.net/siphash/">64-bit 201 * SipHash-2-4 algorithm</a> using a seed value of {@code k = 00 01 02 ...}. 202 * 203 * @since 15.0 204 */ 205 @J2ktIncompatible 206 public static HashFunction sipHash24() { 207 return SipHashFunction.SIP_HASH_24; 208 } 209 210 /** 211 * Returns a hash function implementing the <a href="https://131002.net/siphash/">64-bit 212 * SipHash-2-4 algorithm</a> using the given seed. 213 * 214 * @since 15.0 215 */ 216 @J2ktIncompatible 217 public static HashFunction sipHash24(long k0, long k1) { 218 return new SipHashFunction(2, 4, k0, k1); 219 } 220 221 /** 222 * Returns a hash function implementing the MD5 hash algorithm (128 hash bits). 223 * 224 * @deprecated If you must interoperate with a system that requires MD5, then use this method, 225 * despite its deprecation. But if you can choose your hash function, avoid MD5, which is 226 * neither fast nor secure. As of January 2017, we suggest: 227 * <ul> 228 * <li>For security: 229 * {@link Hashing#sha256} or a higher-level API. 230 * <li>For speed: {@link Hashing#goodFastHash}, though see its docs for caveats. 231 * </ul> 232 */ 233 @Deprecated 234 public static HashFunction md5() { 235 return Md5Holder.MD5; 236 } 237 238 private static final class Md5Holder { 239 static final HashFunction MD5 = new MessageDigestHashFunction("MD5", "Hashing.md5()"); 240 } 241 242 /** 243 * Returns a hash function implementing the SHA-1 algorithm (160 hash bits). 244 * 245 * @deprecated If you must interoperate with a system that requires SHA-1, then use this method, 246 * despite its deprecation. But if you can choose your hash function, avoid SHA-1, which is 247 * neither fast nor secure. As of January 2017, we suggest: 248 * <ul> 249 * <li>For security: 250 * {@link Hashing#sha256} or a higher-level API. 251 * <li>For speed: {@link Hashing#goodFastHash}, though see its docs for caveats. 252 * </ul> 253 */ 254 @Deprecated 255 public static HashFunction sha1() { 256 return Sha1Holder.SHA_1; 257 } 258 259 private static final class Sha1Holder { 260 static final HashFunction SHA_1 = new MessageDigestHashFunction("SHA-1", "Hashing.sha1()"); 261 } 262 263 /** Returns a hash function implementing the SHA-256 algorithm (256 hash bits). */ 264 public static HashFunction sha256() { 265 return Sha256Holder.SHA_256; 266 } 267 268 private static final class Sha256Holder { 269 static final HashFunction SHA_256 = 270 new MessageDigestHashFunction("SHA-256", "Hashing.sha256()"); 271 } 272 273 /** 274 * Returns a hash function implementing the SHA-384 algorithm (384 hash bits). 275 * 276 * @since 19.0 277 */ 278 @J2ktIncompatible 279 public static HashFunction sha384() { 280 return Sha384Holder.SHA_384; 281 } 282 283 private static final class Sha384Holder { 284 static final HashFunction SHA_384 = 285 new MessageDigestHashFunction("SHA-384", "Hashing.sha384()"); 286 } 287 288 /** Returns a hash function implementing the SHA-512 algorithm (512 hash bits). */ 289 @J2ktIncompatible 290 public static HashFunction sha512() { 291 return Sha512Holder.SHA_512; 292 } 293 294 private static final class Sha512Holder { 295 static final HashFunction SHA_512 = 296 new MessageDigestHashFunction("SHA-512", "Hashing.sha512()"); 297 } 298 299 /** 300 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 301 * MD5 (128 hash bits) hash function and the given secret key. 302 * 303 * <p>If you are designing a new system that needs HMAC, prefer {@link #hmacSha256} or other 304 * future-proof algorithms <a 305 * href="https://datatracker.ietf.org/doc/html/rfc6151#section-2.3">over {@code hmacMd5}</a>. 306 * 307 * @param key the secret key 308 * @throws IllegalArgumentException if the given key is inappropriate for initializing this MAC 309 * @since 20.0 310 */ 311 @J2ktIncompatible 312 public static HashFunction hmacMd5(Key key) { 313 return new MacHashFunction("HmacMD5", key, hmacToString("hmacMd5", key)); 314 } 315 316 /** 317 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 318 * MD5 (128 hash bits) hash function and a {@link SecretKeySpec} created from the given byte array 319 * and the MD5 algorithm. 320 * 321 * <p>If you are designing a new system that needs HMAC, prefer {@link #hmacSha256} or other 322 * future-proof algorithms <a 323 * href="https://datatracker.ietf.org/doc/html/rfc6151#section-2.3">over {@code hmacMd5}</a>. 324 * 325 * @param key the key material of the secret key 326 * @since 20.0 327 */ 328 @J2ktIncompatible 329 public static HashFunction hmacMd5(byte[] key) { 330 return hmacMd5(new SecretKeySpec(checkNotNull(key), "HmacMD5")); 331 } 332 333 /** 334 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 335 * SHA-1 (160 hash bits) hash function and the given secret key. 336 * 337 * @param key the secret key 338 * @throws IllegalArgumentException if the given key is inappropriate for initializing this MAC 339 * @since 20.0 340 */ 341 @J2ktIncompatible 342 public static HashFunction hmacSha1(Key key) { 343 return new MacHashFunction("HmacSHA1", key, hmacToString("hmacSha1", key)); 344 } 345 346 /** 347 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 348 * SHA-1 (160 hash bits) hash function and a {@link SecretKeySpec} created from the given byte 349 * array and the SHA-1 algorithm. 350 * 351 * @param key the key material of the secret key 352 * @since 20.0 353 */ 354 @J2ktIncompatible 355 public static HashFunction hmacSha1(byte[] key) { 356 return hmacSha1(new SecretKeySpec(checkNotNull(key), "HmacSHA1")); 357 } 358 359 /** 360 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 361 * SHA-256 (256 hash bits) hash function and the given secret key. 362 * 363 * @param key the secret key 364 * @throws IllegalArgumentException if the given key is inappropriate for initializing this MAC 365 * @since 20.0 366 */ 367 @J2ktIncompatible 368 public static HashFunction hmacSha256(Key key) { 369 return new MacHashFunction("HmacSHA256", key, hmacToString("hmacSha256", key)); 370 } 371 372 /** 373 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 374 * SHA-256 (256 hash bits) hash function and a {@link SecretKeySpec} created from the given byte 375 * array and the SHA-256 algorithm. 376 * 377 * @param key the key material of the secret key 378 * @since 20.0 379 */ 380 @J2ktIncompatible 381 public static HashFunction hmacSha256(byte[] key) { 382 return hmacSha256(new SecretKeySpec(checkNotNull(key), "HmacSHA256")); 383 } 384 385 /** 386 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 387 * SHA-512 (512 hash bits) hash function and the given secret key. 388 * 389 * @param key the secret key 390 * @throws IllegalArgumentException if the given key is inappropriate for initializing this MAC 391 * @since 20.0 392 */ 393 @J2ktIncompatible 394 public static HashFunction hmacSha512(Key key) { 395 return new MacHashFunction("HmacSHA512", key, hmacToString("hmacSha512", key)); 396 } 397 398 /** 399 * Returns a hash function implementing the Message Authentication Code (MAC) algorithm, using the 400 * SHA-512 (512 hash bits) hash function and a {@link SecretKeySpec} created from the given byte 401 * array and the SHA-512 algorithm. 402 * 403 * @param key the key material of the secret key 404 * @since 20.0 405 */ 406 @J2ktIncompatible 407 public static HashFunction hmacSha512(byte[] key) { 408 return hmacSha512(new SecretKeySpec(checkNotNull(key), "HmacSHA512")); 409 } 410 411 private static String hmacToString(String methodName, Key key) { 412 return "Hashing." 413 + methodName 414 + "(Key[algorithm=" 415 + key.getAlgorithm() 416 + ", format=" 417 + key.getFormat() 418 + "])"; 419 } 420 421 /** 422 * Returns a hash function implementing the CRC32C checksum algorithm (32 hash bits) as described 423 * by RFC 3720, Section 12.1. 424 * 425 * <p>This function is best understood as a <a 426 * href="https://en.wikipedia.org/wiki/Checksum">checksum</a> rather than a true <a 427 * href="https://en.wikipedia.org/wiki/Hash_function">hash function</a>. 428 * 429 * @since 18.0 430 */ 431 @J2ktIncompatible 432 public static HashFunction crc32c() { 433 return Crc32CSupplier.HASH_FUNCTION; 434 } 435 436 @Immutable 437 @J2ktIncompatible 438 private enum Crc32CSupplier implements ImmutableSupplier<HashFunction> { 439 @J2ObjCIncompatible 440 JAVA_UTIL_ZIP { 441 @Override 442 public HashFunction get() { 443 return ChecksumType.CRC_32C.hashFunction; 444 } 445 }, 446 ABSTRACT_HASH_FUNCTION { 447 @Override 448 public HashFunction get() { 449 return Crc32cHashFunction.CRC_32_C; 450 } 451 }; 452 453 static final HashFunction HASH_FUNCTION = pickFunction().get(); 454 455 private static Crc32CSupplier pickFunction() { 456 Crc32CSupplier[] functions = values(); 457 458 if (functions.length == 1) { 459 // We're running under J2ObjC. 460 return functions[0]; 461 } 462 463 // We can't refer to JAVA_UTIL_ZIP directly at compile time because of J2ObjC. 464 Crc32CSupplier javaUtilZip = functions[0]; 465 466 try { 467 Class.forName("java.util.zip.CRC32C"); 468 return javaUtilZip; 469 } catch (ClassNotFoundException runningUnderJava8) { 470 return ABSTRACT_HASH_FUNCTION; 471 } 472 } 473 } 474 475 /** 476 * Returns a hash function implementing the CRC-32 checksum algorithm (32 hash bits). 477 * 478 * <p>To get the {@code long} value equivalent to {@link Checksum#getValue()} for a {@code 479 * HashCode} produced by this function, use {@link HashCode#padToLong()}. 480 * 481 * <p>This function is best understood as a <a 482 * href="https://en.wikipedia.org/wiki/Checksum">checksum</a> rather than a true <a 483 * href="https://en.wikipedia.org/wiki/Hash_function">hash function</a>. 484 * 485 * @since 14.0 486 */ 487 @J2ktIncompatible 488 public static HashFunction crc32() { 489 return ChecksumType.CRC_32.hashFunction; 490 } 491 492 /** 493 * Returns a hash function implementing the Adler-32 checksum algorithm (32 hash bits). 494 * 495 * <p>To get the {@code long} value equivalent to {@link Checksum#getValue()} for a {@code 496 * HashCode} produced by this function, use {@link HashCode#padToLong()}. 497 * 498 * <p>This function is best understood as a <a 499 * href="https://en.wikipedia.org/wiki/Checksum">checksum</a> rather than a true <a 500 * href="https://en.wikipedia.org/wiki/Hash_function">hash function</a>. 501 * 502 * @since 14.0 503 */ 504 @J2ktIncompatible 505 public static HashFunction adler32() { 506 return ChecksumType.ADLER_32.hashFunction; 507 } 508 509 @Immutable 510 @J2ktIncompatible 511 enum ChecksumType implements ImmutableSupplier<Checksum> { 512 CRC_32("Hashing.crc32()") { 513 @Override 514 public Checksum get() { 515 return new CRC32(); 516 } 517 }, 518 @J2ObjCIncompatible 519 CRC_32C("Hashing.crc32c()") { 520 // Crc32CSupplier.pickFunction uses this only when it finds that CRC32C is available. 521 @SuppressWarnings("Java8ApiChecker") 522 @IgnoreJRERequirement 523 @Override 524 public Checksum get() { 525 return new CRC32C(); 526 } 527 }, 528 ADLER_32("Hashing.adler32()") { 529 @Override 530 public Checksum get() { 531 return new Adler32(); 532 } 533 }; 534 535 public final HashFunction hashFunction; 536 537 ChecksumType(String toString) { 538 this.hashFunction = new ChecksumHashFunction(this, 32, toString); 539 } 540 } 541 542 /** 543 * Returns a hash function implementing FarmHash's Fingerprint64, an open-source algorithm. 544 * 545 * <p>This is designed for generating persistent fingerprints of strings. It isn't 546 * cryptographically secure, but it produces a high-quality hash with fewer collisions than some 547 * alternatives we've used in the past. 548 * 549 * <p>FarmHash fingerprints are encoded by {@link HashCode#asBytes} in little-endian order. This 550 * means {@link HashCode#asLong} is guaranteed to return the same value that 551 * farmhash::Fingerprint64() would for the same input (when compared using {@link 552 * com.google.common.primitives.UnsignedLongs}'s encoding of 64-bit unsigned numbers). 553 * 554 * <p>This function is best understood as a <a 555 * href="https://en.wikipedia.org/wiki/Fingerprint_(computing)">fingerprint</a> rather than a true 556 * <a href="https://en.wikipedia.org/wiki/Hash_function">hash function</a>. 557 * 558 * @since 20.0 559 */ 560 public static HashFunction farmHashFingerprint64() { 561 return FarmHashFingerprint64.FARMHASH_FINGERPRINT_64; 562 } 563 564 /** 565 * Returns a hash function implementing the Fingerprint2011 hashing function (64 hash bits). 566 * 567 * <p>This is designed for generating persistent fingerprints of strings. It isn't 568 * cryptographically secure, but it produces a high-quality hash with few collisions. Fingerprints 569 * generated using this are byte-wise identical to those created using the C++ version, but note 570 * that this uses unsigned integers (see {@link com.google.common.primitives.UnsignedInts}). 571 * Comparisons between the two should take this into account. 572 * 573 * <p>Fingerprint2011() is a form of Murmur2 on strings up to 32 bytes and a form of CityHash for 574 * longer strings. It could have been one or the other throughout. The main advantage of the 575 * combination is that CityHash has a bunch of special cases for short strings that don't need to 576 * be replicated here. The result will never be 0 or 1. 577 * 578 * <p>This function is best understood as a <a 579 * href="https://en.wikipedia.org/wiki/Fingerprint_(computing)">fingerprint</a> rather than a true 580 * <a href="https://en.wikipedia.org/wiki/Hash_function">hash function</a>. 581 * 582 * @since 31.1 583 */ 584 @J2ktIncompatible 585 public static HashFunction fingerprint2011() { 586 return Fingerprint2011.FINGERPRINT_2011; 587 } 588 589 /** 590 * Assigns to {@code hashCode} a "bucket" in the range {@code [0, buckets)}, in a uniform manner 591 * that minimizes the need for remapping as {@code buckets} grows. That is, {@code 592 * consistentHash(h, n)} equals: 593 * 594 * <ul> 595 * <li>{@code n - 1}, with approximate probability {@code 1/n} 596 * <li>{@code consistentHash(h, n - 1)}, otherwise (probability {@code 1 - 1/n}) 597 * </ul> 598 * 599 * <p>This method is suitable for the common use case of dividing work among buckets that meet the 600 * following conditions: 601 * 602 * <ul> 603 * <li>You want to assign the same fraction of inputs to each bucket. 604 * <li>When you reduce the number of buckets, you can accept that the most recently added 605 * buckets will be removed first. More concretely, if you are dividing traffic among tasks, 606 * you can decrease the number of tasks from 15 and 10, killing off the final 5 tasks, and 607 * {@code consistentHash} will handle it. If, however, you are dividing traffic among 608 * servers {@code alpha}, {@code bravo}, and {@code charlie} and you occasionally need to 609 * take each of the servers offline, {@code consistentHash} will be a poor fit: It provides 610 * no way for you to specify which of the three buckets is disappearing. Thus, if your 611 * buckets change from {@code [alpha, bravo, charlie]} to {@code [bravo, charlie]}, it will 612 * assign all the old {@code alpha} traffic to {@code bravo} and all the old {@code bravo} 613 * traffic to {@code charlie}, rather than letting {@code bravo} keep its traffic. 614 * </ul> 615 * 616 * <p>See the <a href="http://en.wikipedia.org/wiki/Consistent_hashing">Wikipedia article on 617 * consistent hashing</a> for more information. 618 */ 619 @J2ktIncompatible 620 public static int consistentHash(HashCode hashCode, int buckets) { 621 return consistentHash(hashCode.padToLong(), buckets); 622 } 623 624 /** 625 * Assigns to {@code input} a "bucket" in the range {@code [0, buckets)}, in a uniform manner that 626 * minimizes the need for remapping as {@code buckets} grows. That is, {@code consistentHash(h, 627 * n)} equals: 628 * 629 * <ul> 630 * <li>{@code n - 1}, with approximate probability {@code 1/n} 631 * <li>{@code consistentHash(h, n - 1)}, otherwise (probability {@code 1 - 1/n}) 632 * </ul> 633 * 634 * <p>This method is suitable for the common use case of dividing work among buckets that meet the 635 * following conditions: 636 * 637 * <ul> 638 * <li>You want to assign the same fraction of inputs to each bucket. 639 * <li>When you reduce the number of buckets, you can accept that the most recently added 640 * buckets will be removed first. More concretely, if you are dividing traffic among tasks, 641 * you can decrease the number of tasks from 15 and 10, killing off the final 5 tasks, and 642 * {@code consistentHash} will handle it. If, however, you are dividing traffic among 643 * servers {@code alpha}, {@code bravo}, and {@code charlie} and you occasionally need to 644 * take each of the servers offline, {@code consistentHash} will be a poor fit: It provides 645 * no way for you to specify which of the three buckets is disappearing. Thus, if your 646 * buckets change from {@code [alpha, bravo, charlie]} to {@code [bravo, charlie]}, it will 647 * assign all the old {@code alpha} traffic to {@code bravo} and all the old {@code bravo} 648 * traffic to {@code charlie}, rather than letting {@code bravo} keep its traffic. 649 * </ul> 650 * 651 * <p>See the <a href="http://en.wikipedia.org/wiki/Consistent_hashing">Wikipedia article on 652 * consistent hashing</a> for more information. 653 */ 654 @J2ktIncompatible 655 public static int consistentHash(long input, int buckets) { 656 checkArgument(buckets > 0, "buckets must be positive: %s", buckets); 657 long generatorState = input; 658 int candidate = 0; 659 int generated; 660 661 // Jump from bucket to bucket until we go out of range 662 while (true) { 663 generatorState = LinearCongruentialGenerator.nextState(generatorState); 664 generated = (int) ((candidate + 1) / LinearCongruentialGenerator.toDouble(generatorState)); 665 if (generated >= 0 && generated < buckets) { 666 candidate = generated; 667 } else { 668 return candidate; 669 } 670 } 671 } 672 673 /** 674 * Returns a hash code, having the same bit length as each of the input hash codes, that combines 675 * the information of these hash codes in an ordered fashion. That is, whenever two equal hash 676 * codes are produced by two calls to this method, it is <i>as likely as possible</i> that each 677 * was computed from the <i>same</i> input hash codes in the <i>same</i> order. 678 * 679 * @throws IllegalArgumentException if {@code hashCodes} is empty, or the hash codes do not all 680 * have the same bit length 681 */ 682 @J2ktIncompatible 683 public static HashCode combineOrdered(Iterable<HashCode> hashCodes) { 684 Iterator<HashCode> iterator = hashCodes.iterator(); 685 checkArgument(iterator.hasNext(), "Must be at least 1 hash code to combine."); 686 int bits = iterator.next().bits(); 687 byte[] resultBytes = new byte[bits / 8]; 688 for (HashCode hashCode : hashCodes) { 689 byte[] nextBytes = hashCode.getBytesInternal(); 690 checkArgument( 691 nextBytes.length == resultBytes.length, "All hashcodes must have the same bit length."); 692 for (int i = 0; i < nextBytes.length; i++) { 693 resultBytes[i] = (byte) (resultBytes[i] * 37 ^ nextBytes[i]); 694 } 695 } 696 return HashCode.fromBytesNoCopy(resultBytes); 697 } 698 699 /** 700 * Returns a hash code, having the same bit length as each of the input hash codes, that combines 701 * the information of these hash codes in an unordered fashion. That is, whenever two equal hash 702 * codes are produced by two calls to this method, it is <i>as likely as possible</i> that each 703 * was computed from the <i>same</i> input hash codes in <i>some</i> order. 704 * 705 * @throws IllegalArgumentException if {@code hashCodes} is empty, or the hash codes do not all 706 * have the same bit length 707 */ 708 @J2ktIncompatible 709 public static HashCode combineUnordered(Iterable<HashCode> hashCodes) { 710 Iterator<HashCode> iterator = hashCodes.iterator(); 711 checkArgument(iterator.hasNext(), "Must be at least 1 hash code to combine."); 712 byte[] resultBytes = new byte[iterator.next().bits() / 8]; 713 for (HashCode hashCode : hashCodes) { 714 byte[] nextBytes = hashCode.getBytesInternal(); 715 checkArgument( 716 nextBytes.length == resultBytes.length, "All hashcodes must have the same bit length."); 717 for (int i = 0; i < nextBytes.length; i++) { 718 resultBytes[i] += nextBytes[i]; 719 } 720 } 721 return HashCode.fromBytesNoCopy(resultBytes); 722 } 723 724 /** Checks that the passed argument is positive, and ceils it to a multiple of 32. */ 725 static int checkPositiveAndMakeMultipleOf32(int bits) { 726 checkArgument(bits > 0, "Number of bits must be positive"); 727 return (bits + 31) & ~31; 728 } 729 730 /** 731 * Returns a hash function which computes its hash code by concatenating the hash codes of the 732 * underlying hash functions together. This can be useful if you need to generate hash codes of a 733 * specific length. 734 * 735 * <p>For example, if you need 1024-bit hash codes, you could join two {@link Hashing#sha512} hash 736 * functions together: {@code Hashing.concatenating(Hashing.sha512(), Hashing.sha512())}. 737 * 738 * @since 19.0 739 */ 740 @J2ktIncompatible 741 public static HashFunction concatenating( 742 HashFunction first, HashFunction second, HashFunction... rest) { 743 // We can't use Lists.asList() here because there's no hash->collect dependency 744 List<HashFunction> list = new ArrayList<>(); 745 list.add(first); 746 list.add(second); 747 Collections.addAll(list, rest); 748 return new ConcatenatedHashFunction(list.toArray(new HashFunction[0])); 749 } 750 751 /** 752 * Returns a hash function which computes its hash code by concatenating the hash codes of the 753 * underlying hash functions together. This can be useful if you need to generate hash codes of a 754 * specific length. 755 * 756 * <p>For example, if you need 1024-bit hash codes, you could join two {@link Hashing#sha512} hash 757 * functions together: {@code Hashing.concatenating(Hashing.sha512(), Hashing.sha512())}. 758 * 759 * @since 19.0 760 */ 761 @J2ktIncompatible 762 public static HashFunction concatenating(Iterable<HashFunction> hashFunctions) { 763 checkNotNull(hashFunctions); 764 // We can't use Iterables.toArray() here because there's no hash->collect dependency 765 List<HashFunction> list = new ArrayList<>(); 766 for (HashFunction hashFunction : hashFunctions) { 767 list.add(hashFunction); 768 } 769 checkArgument(!list.isEmpty(), "number of hash functions (%s) must be > 0", list.size()); 770 return new ConcatenatedHashFunction(list.toArray(new HashFunction[0])); 771 } 772 773 @J2ktIncompatible 774 private static final class ConcatenatedHashFunction extends AbstractCompositeHashFunction { 775 776 private ConcatenatedHashFunction(HashFunction... functions) { 777 super(functions); 778 for (HashFunction function : functions) { 779 checkArgument( 780 function.bits() % 8 == 0, 781 "the number of bits (%s) in hashFunction (%s) must be divisible by 8", 782 function.bits(), 783 function); 784 } 785 } 786 787 @Override 788 HashCode makeHash(Hasher[] hashers) { 789 byte[] bytes = new byte[bits() / 8]; 790 int i = 0; 791 for (Hasher hasher : hashers) { 792 HashCode newHash = hasher.hash(); 793 i += newHash.writeBytesTo(bytes, i, newHash.bits() / 8); 794 } 795 return HashCode.fromBytesNoCopy(bytes); 796 } 797 798 @Override 799 public int bits() { 800 int bitSum = 0; 801 for (HashFunction function : functions) { 802 bitSum += function.bits(); 803 } 804 return bitSum; 805 } 806 807 @Override 808 public boolean equals(@Nullable Object object) { 809 if (object instanceof ConcatenatedHashFunction) { 810 ConcatenatedHashFunction other = (ConcatenatedHashFunction) object; 811 return Arrays.equals(functions, other.functions); 812 } 813 return false; 814 } 815 816 @Override 817 public int hashCode() { 818 return Arrays.hashCode(functions); 819 } 820 } 821 822 /** 823 * Linear CongruentialGenerator to use for consistent hashing. See 824 * http://en.wikipedia.org/wiki/Linear_congruential_generator 825 */ 826 private static final class LinearCongruentialGenerator { 827 /** Generator state is managed by the caller to avoid unnecessary allocations. */ 828 static long nextState(long state) { 829 return 2862933555777941757L * state + 1; 830 } 831 832 /** Generator state must be updated with {@link #nextState} before calling {@link #toDouble}. */ 833 static double toDouble(long state) { 834 return ((double) ((int) (state >>> 33) + 1)) / 0x1.0p31; 835 } 836 } 837 838 private Hashing() {} 839}