[EOS/CODE] BitN - packing thousands of booleans into tease storage
Posted: Sun Nov 28, 2021 3:05 pm
While SaveState and JsonComp do pretty well at packing a bunch of Booleans and other variables into a relatively small amount of Eos teaseStorage, sometimes you need more true/false states than you have available bytes[1] in storage.
For that, there's BitN. BitN allows you to pack 6 booleans (bits[4]) per character (byte).
Using BitN, you could conceivably store up to ~ 6,084 boolean states in tease storage, if you were to store nothing but a single huge 1014 byte[2] bitmap.
To use BitN, simply copy/paste the following to the top of your Init Script:
(Source Code)
Footnotes:
[1] Eos teaseStorage only gives us 1KB (1024 bytes) for storage, including encoding[2][3] it uses to store the variables/characters you set. This leaves us with a limited set of usable characters we can actually store while only taking up a single byte per character.
[2] teaseSotage encodes all items you store with JSON. If you store, for example:
teaseStorage.setItem("myName", "I am the great and powerful Oz.");
teaseStorage.setItem("things", "I like ducks and cantaloupe.");
... what is actually saved by Eos is:
{"myName":"I am the great and powerful Oz.","things":"I like ducks and cantaloupe."}
... consuming 84 bytes total. So, because of JSON encoding overhead, we don't get to use all 1024 bytes for data.
[3] Some characters that appear to only take one byte in JavaScript will take multiple bytes when stored with teaseStorage, like double quote, back-slash, line-feeds, etc. Other characters, like emojis and ascii 127-159, simply can't be stored directly with Eos, and will blow away any existing storage for that user for the given tease if you try to use them. (JsonComp and SaveState allow storage of all special characters.)
[4] Why just 6 bits per byte? Why not 7 bits, or even 8?
8 bits per byte isn't possible because we can't use all of the 256 characters[3] needed for 8 bit encoding.
7 bits per byte certainly could be done if storing directly to teaseStorage, but I wanted to keep BitN compatible with JsonComp and SaveState. JsonComp uses a great many of the extended Ascii characters (> 127) as symbols for common character combinations and signals in its compression algorithm. As a result, there aren't enough left to fill out the set of 128 characters I'd need for 7 bit encoding without sacrificing compression efficiency. (If there's truly need for a 7 bit version, let me know.)
For that, there's BitN. BitN allows you to pack 6 booleans (bits[4]) per character (byte).
Using BitN, you could conceivably store up to ~ 6,084 boolean states in tease storage, if you were to store nothing but a single huge 1014 byte[2] bitmap.
To use BitN, simply copy/paste the following to the top of your Init Script:
Code: Select all
// Install Bit6 & BitN v0.2.4 (see: https://milovana.com/forum/viewtopic.php?f=2&t=24735)
;(function(){var t="0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@".split(""),n=t.reduce(function(t,n,r){return t[n]=r,t},{}),r=t.reduce(function(t,n,r){for(var o=0,u=0;u<6;u++)o+=r&1<<u?1:0;return t[n]=o,t},{}),o=parseInt("111111",2);function u(t){var r=n[t];if(isNaN(r))throw"Invalid 6 bit char: "+t;return r}function e(n){var r=t[n];if(void 0===r)throw"Out of 6 bit range: "+n;return r}function i(){return t[0]}function f(t,n){return e(u(t)|1<<n)}function a(t,n,r){return e(r?u(t)|1<<n:u(t)&~(1<<n))}function c(t,n){return 0!=(u(t)&1<<n)}function s(t){return r[t]||0}function l(t,n){return e(u(t)&~(1<<n))}function g(t,n){return e(u(t)^1<<n)}var h={decode:u,encode:e,empty:i,set:f,put:a,test:c,count:s,clear:l,toggle:g},v={maxBit:function(t){return 6*t.length-1},bytesFor:function(t){return Math.floor(t/6)+1},pad:function(t,n){for(var r="",o=0,u=Math.floor(n/6)+1-t.length;o<u;o++)r+=i();return t+r},decode:function(t){for(var n="",r=0,o=t.length;r<o;r++)n=("000000"+u(t[r]).toString(2)).slice(-6)+n;return parseInt(n,2)},encode:function(t,n){var r="";void 0===n&&(n=Math.floor(Math.log(t)*Math.LOG2E));for(var u=0,i=Math.floor(n/6);u<=i;u++)r+=e(t&o),t=parseInt(t.toString(2).slice(0,-6),2);return r},empty:function(t){for(var n="",r=0,o=Math.floor(t/6);r<=o;r++)n+=i();return n},set:function(t,n){var r=Math.floor(n/6);return t.substring(0,r)+f(t[r],n-6*r)+t.substring(r+1,t.length)},put:function(t,n,r){var o=Math.floor(n/6);return t.substring(0,o)+a(t[o],n-6*o,r)+t.substring(o+1,t.length)},test:function(t,n){var r=Math.floor(n/6);return c(t[r],n-6*r)},count:function(t){for(var n=0,r=0,o=t.length;r<o;r++)n+=s(t[r]);return n},clear:function(t,n){var r=Math.floor(n/6);return t.substring(0,r)+l(t[r],n-6*r)+t.substring(r+1,t.length)},toggle:function(t,n){var r=Math.floor(n/6);return t.substring(0,r)+g(t[r],n-6*r)+t.substring(r+1,t.length)}};window.Bit6=h,window.BitN=v})()
- Spoiler: show
Code: Select all
// Create a variable with bits 0 to 11 (12 bits)
// Note: Each 6 bits takes one byte in BitN, so empty(11) will take two, as would empty(6) through empty(10)
var myTinyBitmap = BitN.empty(11)
// Set some bits:
myTinyBitmap = BitN.set(myTinyBitmap, 0) // set bit 0
myTinyBitmap = BitN.set(myTinyBitmap, 7) // set bit 7
// Test some bits:
if (BitN.test(myTinyBitmap, 7)) {
console.log('Bit 7 is set.')
} else {
console.log('Bit 7 is not set.')
}
// Clear some bits:
myTinyBitmap = BitN.clear(myTinyBitmap, 7) // clear bit 7
// Toggle some bits
myTinyBitmap = BitN.toggle(myTinyBitmap, 7) // If bit 7 was set, it is now clear. If it was clear, it is now set.
// Put some truthy values to bits
myTinyBitmap = BitN.put(myTinyBitmap, 9, ("I am open").match(/open/)) // Set bit 9 if expression is truthy, clear if not
// Count some bits:
var myCountOfSetBits = BitN.count(myTinyBitmap)
console.log('myTinyBitmap has ' + myCountOfSetBits + ' bits set')
// Fondle some bits:
// ;)
// Convert BitN to integer:
var myIntOfBitmap = BitN.decode(myTinyBitmap)
// Convert integer to BitN
var myEncodedIntBitmap = BitN.encode(myIntOfBitmap, 11) // "11" will encode the int to two bytes. (12 bits.) If that's not enough to contain the binary data in the given int, then that data will be lost.
// Check how many bytes (not including any encoding overhead depending on how you end up storing this variable) your BitN uses:
console.log('myTinyBitmap uses ' + myTinyBitmap.length + ' bytes')
// Get the maximum bit position that could be set:
var myMaxBit = BitN.maxBit(myTinyBitmap)
// Get number of bytes required for a given bit position (bit 0 through n)
var neededBytes = BitN.bytesFor(12)
// To make sure a BitN map has enough space for a given number of bits:
var myLargerBitmap = BitN.pad(myTinyBitmap, 29) // Will return a five byte bitmap, storing up to 30 bits
// DON'T DO THINGS LIKE THIS!
// In this example, myTinyBitmap is only two bytes because it was created with .empty(11).
// Trying to set a bit position higher than 11 will fail.
// (However, if myTinyBitmap was created with empty(12), the following would work, because (12/6) + 1 = 3 bytes)
myTinyBitmap = BitN.toggle(myTinyBitmap, 14) // THIS WILL FAIL ON A TWO BYTE BITMAP!
[1] Eos teaseStorage only gives us 1KB (1024 bytes) for storage, including encoding[2][3] it uses to store the variables/characters you set. This leaves us with a limited set of usable characters we can actually store while only taking up a single byte per character.
[2] teaseSotage encodes all items you store with JSON. If you store, for example:
teaseStorage.setItem("myName", "I am the great and powerful Oz.");
teaseStorage.setItem("things", "I like ducks and cantaloupe.");
... what is actually saved by Eos is:
{"myName":"I am the great and powerful Oz.","things":"I like ducks and cantaloupe."}
... consuming 84 bytes total. So, because of JSON encoding overhead, we don't get to use all 1024 bytes for data.
[3] Some characters that appear to only take one byte in JavaScript will take multiple bytes when stored with teaseStorage, like double quote, back-slash, line-feeds, etc. Other characters, like emojis and ascii 127-159, simply can't be stored directly with Eos, and will blow away any existing storage for that user for the given tease if you try to use them. (JsonComp and SaveState allow storage of all special characters.)
[4] Why just 6 bits per byte? Why not 7 bits, or even 8?
8 bits per byte isn't possible because we can't use all of the 256 characters[3] needed for 8 bit encoding.
7 bits per byte certainly could be done if storing directly to teaseStorage, but I wanted to keep BitN compatible with JsonComp and SaveState. JsonComp uses a great many of the extended Ascii characters (> 127) as symbols for common character combinations and signals in its compression algorithm. As a result, there aren't enough left to fill out the set of 128 characters I'd need for 7 bit encoding without sacrificing compression efficiency. (If there's truly need for a 7 bit version, let me know.)