So you’ve written your gists, and used them to automate tasks. But when you’re changing them, how do you know if they’re getting better and not worse?Gists is the first platform that makes it easy to create test cases for your gists, so you can ensure consistency when you are changing prompts.This can cut your prompt-engineering time by over 52%.
A test case consists of values for all the variables used in a gist, as well as an expected output as the reference.For example, if you have the following gist:
Classify the following tweet into one of these categories: [news, politics, meme, sports, business, celebrity, technology]Tweet: """{{tweet}}"""
We could add the following test cases:
tweet: Reese Witherspoon Tears Up Saying She Felt Like She "Broke" a Year Agoexpected: celebrity
tweet: Amazon's Andy Jassy Plans to Crash the AI Partyexpected: technology
Note, LLMs could classify the last tweet as either technology or business. Which is why we need test cases to understand the behaviors of LLM prompts.
By defining test cases for your gists, we can now measure the consistency of gists by running them multiple times and calculate the success rates.To calculate the success rates, however, we have to first define how the outputs are evaluated for success and failure.Let’s take a look at the default evaluators.
Was this page helpful?
{"docsConfig":{"$schema":"https://mintlify.com/docs.json","theme":"mint","name":"Gists Doc","colors":{"primary":"#3753EB","light":"#AE87FF","dark":"#0D001D"},"favicon":"/favicon.png","logo":{"dark":"https://mintlify.s3-us-west-1.amazonaws.com/cuelyinc/logo/dark.svg","light":"https://mintlify.s3-us-west-1.amazonaws.com/cuelyinc/logo/light.svg"},"api":{"openapi":"/openapi.yml","playground":{"display":"none"},"mdx":{"server":"https://api.gists.ai/v1","auth":{"method":"key","name":"GISTS-API-KEY"}}},"background":{"decoration":null,"color":{"light":null,"dark":"#090014"}},"navbar":{"links":[{"label":"Home","href":"https://gists.ai"},{"label":"Support","href":"mailto:team@gists.ai"}],"primary":{"type":"button","label":"Dashboard","href":"https://app.gists.ai"}},"footer":{"socials":{"twitter":"https://twitter.com/gistsai","linkedin":"https://www.linkedin.com/company/gists"}},"integrations":{"gtm":{"tagId":"GTM-PM7S2JD8"},"intercom":{"appId":"p3p3z10k"}},"navigation":{"pages":[]}},"deployment":{"deploymentMetadataValue":{"subdomain":"cuelyinc","hasAudioNarration":false,"actualSubdomain":"cuelyinc","customDomains":["docs.gists.ai"],"gitSource":{"type":"github","owner":"gistsai","repo":"docs","deployBranch":"main","contentDirectory":"","isPrivate":false,"mountPath":"","label":""},"chroma":{"liveDeploymentHistoryId":"6a20c3e24a41910615812afd","collectionId":"052da08b-b4cf-4319-a223-9c37fa2ae72d","schemaVersion":"v2"},"feedback":{"thumbs":true},"entitlements":{},"buildId":"6a20c3e24a41910615812afd:success","clientVersion":"0.0.3965","searchFilterCounts":[{"language":"default","tag":"Documentation","count":11},{"language":"default","tag":"Essentials","count":8},{"language":"default","tag":"API Reference","count":3},{"language":"default","tag":"Get Started","count":2},{"language":"default","tag":"API Documentation","count":1},{"language":"default","tag":"Generate","count":1},{"language":"default","tag":"Me","count":1},{"language":"default","tag":"Guides","count":1}]},"assistantConfig":{},"hasChatPermissions":false,"hasWebMcpPermissions":false,"toggles":[{"name":"kill-isr","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-mintlify-slack","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-v7-file-tree","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"agent-integrations-settings","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-pr-grouping","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"custom-prefix","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"analytics-v7","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"user-flows-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"no-publish-flow","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workos-onboarding","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"llms-txt-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"pause-universal-search-reconcile","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving-live","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"stripTemperatureMiddleware","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"per-site-search-delta","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"support-platform-pylon-zendesk","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"snapshot-serving","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"instant-rollback","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications-server","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"granular-notifications","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-private-page-search","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automations-fable","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fixed-ai-action-pricing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"automation-resolve-merge-conflict","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"scoped-publishing","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"mintql","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"page-level-analytics","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-version-history-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-nav-generations","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-page-v2","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"workspace-projects","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"assistant-v5","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"byok-sep-25","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"fork-pr-previews","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"revalidation-jwt","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"vite-origin","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"editor-agent-model-picker","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"billing-updated-9-15","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"hobby-abuse-scan","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false},{"name":"dashboard-inbox","enabled":true,"variant":{"name":"disabled","enabled":false,"feature_enabled":true},"impressionData":false}],"isAtlas":false,"googleFontsInlined":false},"customCss":[],"page":{"pagePath":"essentials/test-cases","slug":"/essentials/test-cases","pageMetadata":{"title":"Test Cases","description":null,"href":"/essentials/test-cases"},"pageType":"default","mdx":{"mdxSource":{"compiledSource":"\"use strict\";const{Fragment:_Fragment,jsx:_jsx,jsxs:_jsxs}=arguments[0];const{useMDXComponents:_provideComponents}=arguments[0];function _createMdxContent(props){const _components={a:`a`,code:`code`,p:`p`,pre:`pre`,span:`span`,..._provideComponents(),...props.components},{CodeBlock,Heading,Tip}=_components;if(!CodeBlock)_missingMdxReference(`CodeBlock`,true);if(!Heading)_missingMdxReference(`Heading`,true);if(!Tip)_missingMdxReference(`Tip`,true);return _jsxs(_Fragment,{children:[_jsxs(_components.p,{children:[`So you’ve written your `,_jsx(_components.a,{href:`/essentials/gists`,children:`gists`}),`, and used them to automate tasks. But when you’re changing them, how do you know if they’re getting better and not worse?`]}),`\n`,_jsx(_components.p,{children:`Gists is the first platform that makes it easy to create test cases for your gists, so you can ensure consistency when you are changing prompts.`}),`\n`,_jsx(_components.p,{children:`This can cut your prompt-engineering time by over 52%.`}),`\n`,_jsx(Heading,{level:`2`,id:`how-it-works`,children:`How it works`}),`\n`,_jsx(_components.p,{children:`A test case consists of values for all the variables used in a gist, as well as an expected output as the reference.`}),`\n`,_jsx(_components.p,{children:`For example, if you have the following gist:`}),`\n`,_jsx(CodeBlock,{numberOfLines:`6`,language:`text`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`text`,children:_jsxs(_components.code,{language:`text`,numberOfLines:`6`,children:[_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`Classify the following tweet into one of these categories: [news, politics, meme, sports, business, celebrity, technology]`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`Tweet: \"\"\"`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`{{tweet}}`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`\"\"\"`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{})})]})})}),`\n`,_jsx(_components.p,{children:`We could add the following test cases:`}),`\n`,_jsx(CodeBlock,{numberOfLines:`3`,language:`text`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`text`,children:_jsxs(_components.code,{language:`text`,numberOfLines:`3`,children:[_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`tweet: Reese Witherspoon Tears Up Saying She Felt Like She \"Broke\" a Year Ago`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`expected: celebrity`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{})})]})})}),`\n`,_jsx(CodeBlock,{numberOfLines:`3`,language:`text`,children:_jsx(_components.pre,{className:`shiki shiki-themes github-light-default dark-plus`,style:{backgroundColor:`#ffffff`,\"--shiki-dark-bg\":`#0B0C0E`,color:`#1f2328`,\"--shiki-dark\":`#D4D4D4`},language:`text`,children:_jsxs(_components.code,{language:`text`,numberOfLines:`3`,children:[_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`tweet: Amazon's Andy Jassy Plans to Crash the AI Party`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{children:`expected: technology`})}),`\n`,_jsx(_components.span,{className:`line`,children:_jsx(_components.span,{})})]})})}),`\n`,_jsx(Tip,{children:`Note, LLMs could classify the last tweet as either technology or business. Which is why we need test cases to understand the behaviors of LLM prompts.`}),`\n`,_jsx(Heading,{level:`2`,id:`running-test-cases`,children:`Running test cases`}),`\n`,_jsx(_components.p,{children:`By defining test cases for your gists, we can now measure the consistency of gists by running them multiple times and calculate the success rates.`}),`\n`,_jsx(_components.p,{children:`To calculate the success rates, however, we have to first define how the outputs are evaluated for success and failure.`}),`\n`,_jsxs(_components.p,{children:[`Let’s take a look at the default `,_jsx(_components.a,{href:`/essentials/evaluators`,children:`evaluators`}),`.`]})]})}function MDXContent(props={}){const{wrapper:MDXLayout}={..._provideComponents(),...props.components};return MDXLayout?_jsx(MDXLayout,{...props,children:_jsx(_createMdxContent,{...props})}):_createMdxContent(props)}return{default:MDXContent};function _missingMdxReference(id,component){throw new Error(`Expected `+(component?`component`:`object`)+\" `\"+id+\"` to be defined: you likely forgot to import, pass, or provide it.\")}","frontmatter":{},"scope":{"config":{},"pageMetadata":{"title":"Test Cases","description":null,"href":"/essentials/test-cases"}}},"mdxSourceWithNoJs":null},"mdxExtracts":{"tableOfContents":[{"title":"How it works","slug":"how-it-works","depth":2,"children":[]},{"title":"Running test cases","slug":"running-test-cases","depth":2,"children":[]}],"codeExamples":{}},"jsLinks":[],"subdomain":"cuelyinc","siteName":"Gists Doc","title":"Test Cases - Gists Doc","navMaps":{"firstHrefInVersion":{},"firstHrefInLanguage":{}},"originalFileLocation":"essentials/test-cases.mdx"},"headHtml":"\u003cscript data-mint-head>(function(){var d=document.documentElement,p=null;if(!p){try{p=localStorage.getItem(\"isDarkMode\")}catch(e){}if(p==='true')p='dark';else if(p==='false')p='light';if(p!=='light'\u0026\u0026p!=='dark'\u0026\u0026p!=='system')p=\"system\"}var r=p==='system'?(matchMedia(\"(prefers-color-scheme: dark)\").matches?'dark':'light'):p;d.classList.remove(r==='dark'?'light':'dark');d.classList.add(r);d.setAttribute(\"data-theme-preference\",p)})();\u003c/script>\u003cmeta data-mint-head name=\"application-name\" content=\"Gists Doc\">\u003cmeta data-mint-head name=\"generator\" content=\"Mintlify\">\u003clink data-mint-head rel=\"apple-touch-icon\" type=\"image/png\" sizes=\"180x180\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/cuelyinc/_generated/favicon/apple-touch-icon.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/cuelyinc/_generated/favicon/favicon-32x32.png?v=3\">\u003clink data-mint-head rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/cuelyinc/_generated/favicon/favicon-16x16.png?v=3\">\u003clink data-mint-head rel=\"shortcut icon\" type=\"image/x-icon\" href=\"https://mintlify.s3.us-west-1.amazonaws.com/cuelyinc/_generated/favicon/favicon.ico?v=3\">\u003cmeta data-mint-head property=\"og:title\" content=\"Test Cases - Gists Doc\">\u003cmeta data-mint-head property=\"og:site_name\" content=\"Gists Doc\">\u003cmeta data-mint-head property=\"og:url\" content=\"https://docs.gists.ai/essentials/test-cases\">\u003cmeta data-mint-head property=\"og:type\" content=\"website\">\u003cmeta data-mint-head property=\"og:image\" content=\"https://cuelyinc.mintlify.app/_mintlify/api/og/v2?division=Essentials\u0026amp;title=Test+Cases\u0026amp;theme=a78ff3b2d65ca17052bed1c3\u0026amp;signature=a28a2951e7d6e21f34347c944bd3217414d53d3db712544a8d4f3b47b5c1adaa\">\u003cmeta data-mint-head property=\"og:image:width\" content=\"1200\">\u003cmeta data-mint-head property=\"og:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"twitter:card\" content=\"summary_large_image\">\u003cmeta data-mint-head name=\"twitter:title\" content=\"Test Cases - Gists Doc\">\u003cmeta data-mint-head name=\"twitter:image\" content=\"https://cuelyinc.mintlify.app/_mintlify/api/og/v2?division=Essentials\u0026amp;title=Test+Cases\u0026amp;theme=a78ff3b2d65ca17052bed1c3\u0026amp;signature=a28a2951e7d6e21f34347c944bd3217414d53d3db712544a8d4f3b47b5c1adaa\">\u003cmeta data-mint-head name=\"twitter:image:width\" content=\"1200\">\u003cmeta data-mint-head name=\"twitter:image:height\" content=\"630\">\u003cmeta data-mint-head name=\"msapplication-config\" content=\"https://mintlify.s3.us-west-1.amazonaws.com/cuelyinc/_generated/favicon/browserconfig.xml?v=3\">\u003cmeta data-mint-head name=\"apple-mobile-web-app-title\" content=\"Gists Doc\">\u003cmeta data-mint-head name=\"msapplication-TileColor\" content=\"#3753EB\">\u003clink data-mint-head rel=\"canonical\" href=\"https://docs.gists.ai/essentials/test-cases\">\u003clink data-mint-head rel=\"alternate\" type=\"application/xml\" href=\"/sitemap.xml\">\u003clink data-mint-head rel=\"alternate\" type=\"text/markdown\" href=\"/essentials/test-cases.md\">\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@graph\":[{\"@type\":\"Organization\",\"@id\":\"https://docs.gists.ai/#organization\",\"name\":\"Gists Doc\",\"url\":\"https://docs.gists.ai\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://mintlify.s3-us-west-1.amazonaws.com/cuelyinc/logo/light.svg\"}},{\"@type\":\"WebSite\",\"@id\":\"https://docs.gists.ai#website\",\"name\":\"Gists Doc\",\"url\":\"https://docs.gists.ai\",\"publisher\":{\"@id\":\"https://docs.gists.ai/#organization\"}},{\"@type\":\"WebPage\",\"@id\":\"https://docs.gists.ai/essentials/test-cases#webpage\",\"url\":\"https://docs.gists.ai/essentials/test-cases\",\"name\":\"Test Cases\",\"isPartOf\":{\"@id\":\"https://docs.gists.ai#website\"},\"breadcrumb\":{\"@id\":\"https://docs.gists.ai/essentials/test-cases#breadcrumb\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https://docs.gists.ai/essentials/test-cases#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Essentials\",\"item\":\"https://docs.gists.ai/essentials/openai\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Test Cases\",\"item\":\"https://docs.gists.ai/essentials/test-cases\"}]},{\"@type\":[\"Article\",\"TechArticle\"],\"@id\":\"https://docs.gists.ai/essentials/test-cases#article\",\"headline\":\"Test Cases\",\"name\":\"Test Cases\",\"url\":\"https://docs.gists.ai/essentials/test-cases\",\"mainEntityOfPage\":{\"@id\":\"https://docs.gists.ai/essentials/test-cases#webpage\"},\"publisher\":{\"@id\":\"https://docs.gists.ai/#organization\"},\"isPartOf\":{\"@id\":\"https://docs.gists.ai#website\"}}]}\u003c/script>\u003cscript data-mint-head type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"WebSite\",\"name\":\"Gists Doc\",\"creator\":{\"@type\":\"Organization\",\"name\":\"Mintlify\",\"url\":\"https://mintlify.com\"}}\u003c/script>","siteName":"Gists Doc","nav":{"global":null,"tabs":[{"tab":"Documentation","groups":[{"group":"Get Started","pages":[{"title":"Overview of Gists","href":"/introduction"},{"title":"Quick start","href":"/quickstart"}]},{"group":"Essentials","pages":[{"title":"OpenAI","href":"/essentials/openai"},{"title":"Prompts","href":"/essentials/prompts"},{"title":"Gists","href":"/essentials/gists"},{"title":"Variables","href":"/essentials/variables"},{"title":"Variants","description":null,"href":"/essentials/variants"},{"title":"Test Cases","href":"/essentials/test-cases"},{"title":"Evaluators","description":null,"href":"/essentials/evaluators"},{"title":"Chats","href":"/essentials/chats"},{"title":"Characters","href":"/essentials/characters"},{"title":"Benchmarks","href":"/essentials/benchmarks"}]},{"group":"Guides","pages":[{"title":"Chat","href":"/guides/chat"},{"title":"Gist editor","href":"/guides/gist-editor"},{"title":"Characters","href":"/guides/characters"},{"title":"Desktop","href":"/guides/desktop"},{"title":"Extension","href":"/guides/extension"},{"title":"Api","href":"/guides/api"},{"title":"Using Gists in Zapier","href":"/guides/zapier"},{"title":"Slack","href":"/guides/slack"}]},{"group":"Gist Recipes","pages":[{"title":"Classification","href":"/recipes/classification"},{"title":"Summarization","href":"/recipes/summarization"},{"title":"Brand voice","href":"/recipes/brand-voice"},{"title":"Information extraction","href":"/recipes/information-extraction"},{"title":"Draft generation","href":"/recipes/draft-generation"}]}]},{"tab":"API Reference","pages":[{"title":"API Reference","href":"/api-reference/authentication"}]}]},"title":"Test Cases - Gists Doc","buildId":"dpl_849ArpqvBiCczQiT1Y1stNGpbqHT","deploymentId":"dpl_849ArpqvBiCczQiT1Y1stNGpbqHT","dynamics":[],"chunks":[]}
