How to Prevent Argument too large error in Google Apps Script

In Google Apps Script, we can store data in properties and cache. Both come with their own limitations. A property can hold a maximum of up to 9KB as mentioned on Quotas for Google Services page. Whereas we can store a maximum of up to 100KB data per key in cache.

In some cases, the data you try to store in a property or cache key exceeds in size and your code throws Argument too large error. There is a workaround which lets you store more data per property/cache key. I use that piece of code in the add-ons I have developed.

Three functions are involved in this workaround. The first function converts the data you pass into a JSON string and splits it into chunks and stores it in multiple keys. Second function stitches and parses the data we passed in the first step and returns the original data. The third function deletes the data we saved from the property/cache store.

Other than the three above mentioned functions, there are a lot more utility functions used in this process. You can check the comments in the code to know what it does.

Three different functions to get, set, delete your data are saveInBatches, retrieveFromBatchesand deleteBatches respectively. Arguments of the utility functions are explained below.

• saveInBatches accepts name, value, type, scope. Name is the key you will use to access your data, value is the data you want to store, type is either propery / cache, scope is one of user, document, script.
• retrieveFromBatches & deleteBatches accepts name, type, scope.

Entire code is shared below. You can also access the code as a gist. You also need to keep in mind that, you can store a maximum of up to 500KB / store when the storage type is property. If you exceed that, you will get Exception: You have exceeded the property storage quota. Please remove some properties and try again. error.

/**
 * Function that gets data from or sets data to the property / cache.
 *
 * @param {String} storage property / cache (required).
 * @param {String} key to be used to get data stored in the property / cache (required).
 * @param {String} scope of the storage. valid scopes - user, script, document (required).
 * @param {String} dataType of the property / cache, possible values: json, bool, string (required when action is set).
 * @param {String} action to be performed, get / set the property / cache (required if the action is set).
 * @param {String} value if action === 'set', value to be set. (required if the action is set)
 * @param {number} expirationDuration  after which the cache expires (optional)
 * @return the data stored in the property / cache
 */
const getSetStorage = (storage, key, scope, dataType, action, value, expirationDuration = 21600) => {
  let store = storage === 'cache' ? CacheService : PropertiesService;
  let val;

  if (scope === 'user') {
    store = store[storage === 'cache' ? 'getUserCache' : 'getUserProperties']();
  } else if (scope === 'script') {
    store = store[storage === 'cache' ? 'getScriptCache' : 'getScriptProperties']();
  } else {
    store = store[storage === 'cache' ? 'getDocumentCache' : 'getDocumentProperties']();
  }

  if (action === 'set') {
    val = value;
    const argsTobePassed = [key, dataType === 'json' ? JSON.stringify(value) : value];

    if (storage === 'cache') argsTobePassed.push(expirationDuration);

    store[storage === 'cache' ? 'put' : 'setProperty'](...argsTobePassed);
  } else {
    const storedValue = store[storage === 'cache' ? 'get' : 'getProperty'](key);
    if (!storedValue) return null;

    if (dataType === 'json') val = JSON.parse(storedValue);
    else if (dataType === 'bool') val = storedValue === 'true';
    else val = storedValue;
  }

  return val;
};

/**
 * Converts string to byte length
 *
 * @param {String} str string value to be converted to size
 * @return {Number} byte size of the string passed
 */
const byteLength = (str) => {
  // returns the byte length of an utf8 string
  let s = str.length;
  for (let i = str.length - 1; i >= 0; i -= 1) {
    const code = str.charCodeAt(i);
    if (code > 0x7f && code <= 0x7ff) s += 1;
    else if (code > 0x7ff && code <= 0xffff) s += 2;
    if (code >= 0xdc00 && code <= 0xdfff) i -= 1; // trail surrogate
  }
  return s;
};

/**
 * Returns the length of the string to save in the stroage based on its limitation
 *
 * @param {Object / Array} value to be stored
 * @param {String} type cache / property
 * @return {Number} length of the string
 */
const getChunkSize = (value, type) => {
  // Sets the max limit of a chunk. For cache, it is 95KB and for property it is 8.5KB
  const sizeLimit = (type === 'cache' ? 95 : 8.5) * 1024;

  let newValue = value;
  let byteSize = byteLength(newValue);

  while (byteSize > sizeLimit) {
    // Slices the string in half till it is within the size limit
    newValue = value.slice(0, (newValue.length / 2));
    byteSize = byteLength(newValue);
  }

  return newValue.length;
};

/**
 * Saves the property without the Argument too large error
 *
 * @param {String} name of the property
 * @param {Object / Array} value of the property
 * @param {String} type of the storage property / cache
 * @param {String} scope of the storage user / document / script
 */
export const saveInBatches = (name, value, type = 'property', scope = 'user') => {
  const jsonStr = JSON.stringify(value);
  const strSliceLength = getChunkSize(jsonStr, type);

  // Number of iteration to perform
  const totalChunkedIterations = Math.ceil(jsonStr.length / strSliceLength);
  let counter = 0;

  for (let i = 0; i < jsonStr.length; i += strSliceLength) {
    const prop = jsonStr.slice(i, i + strSliceLength);

    getSetStorage(type, `${name}_${counter}`, scope, 'string', 'set', prop);
    counter += 1;
  }

  // Stores the total number of chunks stored to be used when retrieving
  getSetStorage(type, `${name}_total`, scope, 'number', 'set', totalChunkedIterations);
};

/**
 * Retreive the property in batches to avoid Argument too large error
 *
 * @param {String} name of the property
 * @param {String} type of the storage property / cache
 * @param {String} scope of the storage user / document / script
 * @return saved value in the property after fetching from batches
 */
export const retrieveFromBatches = (name, type = 'property', scope = 'user') => {
  const count = Number(getSetStorage(type, `${name}_total`, scope, 'number')) || 0;
  if (count === 0) return undefined;

  let jsonStr = '';

  for (let i = 0; i < count; i += 1) {
    const tempStr = getSetStorage(type, `${name}_${i}`, scope, 'string');

    // JSON string is stitched here
    jsonStr += tempStr;
  }

  // JSON string is parsed and returned
  return JSON.parse(jsonStr);
};

/**
 * Deletes all the saved batches
 *
 * @param {String} name of the property
 * @param {String} type of the storage property / cache
 * @param {String} scope of the storage user / document / script
 */
export const deleteBatches = (name, type = 'property', scope = 'user') => {
  const count = Number(getSetStorage(type, `${name}_total`, scope, 'number')) || 0;
  const serviceFunc = type === 'property' ? PropertiesService : CacheService;

  // returns the method to be used based on the type of storage and scope
  const funcMethod = {
    property: {
      user: 'getUserProperties',
      document: 'getDocumentProperties',
      script: 'getScriptProperties',
    },
    cache: {
      user: 'getUserCache',
      document: 'getDocumentCache',
      script: 'getScriptCache',
    },
  }[type][scope];
  const deleteObj = serviceFunc[funcMethod]();
  const deleteMethod = type === 'property' ? 'deleteProperty' : 'remove';

  for (let i = 0; i < count; i += 1) {
    deleteObj[deleteMethod](`${name}_${i}`);
  }

  deleteObj[deleteMethod](`${name}_total`);
};