Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .bumpy/array-value-types.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"@env-spec/parser": minor
varlock: minor
env-spec-language: minor
---

Add `@type=array(...)` for list-valued config items with per-element validation, array literal values, JSON and comma-separated input, and `T[]` type generation.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My debate internally had been whether to do wrap the type in array() or to split it into a new decorator --
@type=string(startsWith=abc) @arrayType(minLength=2)

Obviously wrapping is a bit more correct, but since our type stuff doesnt actually use normal function calls like the rest of the system I thought it might be cleaner. Also do we want to support nested arrays?

Im not set on it, but curious what you think.

Also - we may as well support object (record/dictionary) too with this work? I dont think arbitrary shaped objects like we had in dmno, but just setting the value type, and optionally the key type.
dictionary(uuid, keyType=string(isLength=2), minKeyCount=2)

2 changes: 1 addition & 1 deletion packages/env-spec-parser/grammar.peggy
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ ConfigItem =

ConfigItemKey = $([a-zA-Z_] [a-zA-Z0-9_.-]*)

ConfigItemValue = FunctionCall / multiLineString / quotedString / unquotedString
ConfigItemValue = FunctionCall / ArrayLiteral / multiLineString / quotedString / unquotedString

CommentBlock =
// not sure if we want to treat these as decorators if within comment block?
Expand Down
8 changes: 4 additions & 4 deletions packages/env-spec-parser/src/classes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,11 +378,11 @@ export class ParsedEnvSpecBlankLine {


export class ParsedEnvSpecConfigItem {
value: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | undefined;
value: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | ParsedEnvSpecArrayLiteral | undefined;

constructor(public data: {
key: string;
value: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | undefined;
value: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | ParsedEnvSpecArrayLiteral | undefined;
preComments: Array<ParsedEnvSpecDecoratorComment | ParsedEnvSpecComment>;
postComment: ParsedEnvSpecDecoratorComment | ParsedEnvSpecComment | undefined;
_location?: any;
Expand All @@ -393,8 +393,8 @@ export class ParsedEnvSpecConfigItem {
throw new Error('Nested key-value pair found in config item');
} else if (expanded instanceof ParsedEnvSpecFunctionArgs) {
throw new Error('Top-level config item cannot be a bare function args');
} else if (expanded instanceof ParsedEnvSpecObjectLiteral || expanded instanceof ParsedEnvSpecArrayLiteral) {
throw new Error('Top-level config item value cannot be a bare object or array literal');
} else if (expanded instanceof ParsedEnvSpecObjectLiteral) {
throw new Error('Top-level config item value cannot be a bare object literal');
}
this.value = expanded;
}
Expand Down
24 changes: 20 additions & 4 deletions packages/env-spec-parser/src/simple-resolver.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { execSync } from 'node:child_process';
import {
ParsedEnvSpecFile, ParsedEnvSpecFunctionCall, ParsedEnvSpecKeyValuePair,
ParsedEnvSpecStaticValue,
ParsedEnvSpecStaticValue, ParsedEnvSpecArrayLiteral,
} from './classes.js';

type SimpleResolvedValue = string | Array<unknown> | undefined;

/**
* very simple resolver meant to be used for testing
* not currently exposed as part of public API
Expand All @@ -18,9 +20,20 @@ export function simpleResolver(
const resolved = {} as Record<string, any>;

function valueResolver(
valOrFn: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall,
): string | undefined {
valOrFn: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | ParsedEnvSpecArrayLiteral,
): SimpleResolvedValue {
if (valOrFn instanceof ParsedEnvSpecStaticValue) return valOrFn.unescapedValue;
if (valOrFn instanceof ParsedEnvSpecArrayLiteral) {
return valOrFn.values.map((v) => {
if (v instanceof ParsedEnvSpecStaticValue || v instanceof ParsedEnvSpecFunctionCall) {
return valueResolver(v);
}
if (v instanceof ParsedEnvSpecArrayLiteral) {
return valueResolver(v);
}
throw new Error('Unsupported array literal element');
});
}
if (valOrFn instanceof ParsedEnvSpecFunctionCall) {
if (valOrFn.name === 'ref') {
const args = valOrFn.simplifiedArgs;
Expand All @@ -38,11 +51,13 @@ export function simpleResolver(
return valueResolver(i);
} else if (i instanceof ParsedEnvSpecFunctionCall) {
return valueResolver(i);
} else if (i instanceof ParsedEnvSpecArrayLiteral) {
return valueResolver(i);
} else {
throw new Error('Invalid concat args');
}
});
return resolvedArgs.join('');
return resolvedArgs.map((v) => (v === undefined ? '' : String(v))).join('');
} else if (valOrFn.name === 'exec') {
const args = valOrFn.simplifiedArgs;
if (Array.isArray(args)) {
Expand Down Expand Up @@ -71,6 +86,7 @@ export function simpleResolver(
if (
!(args[0] instanceof ParsedEnvSpecStaticValue)
&& !(args[0] instanceof ParsedEnvSpecFunctionCall)
&& !(args[0] instanceof ParsedEnvSpecArrayLiteral)
) throw new Error('Expected first arg to be a static value or function call');
const val = valueResolver(args[0]);
const remainingArgs = args.slice(1);
Expand Down
18 changes: 17 additions & 1 deletion packages/env-spec-parser/test/values.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { describe, it, expect } from 'vitest';
import ansis from 'ansis';
import {
ParsedEnvSpecFunctionCall, ParsedEnvSpecKeyValuePair,
ParsedEnvSpecStaticValue, parseEnvSpecDotEnvFile,
ParsedEnvSpecStaticValue, ParsedEnvSpecArrayLiteral, parseEnvSpecDotEnvFile,
} from '../src';
import { expectInstanceOf } from './test-utils';

Expand Down Expand Up @@ -225,3 +225,19 @@ describe('regex-like strings and paths with slashes', () => {
expect(args[0].value.value).toEqual('/^https:\\/\\//');
});
});

describe('array literal item values', () => {
it('parses [a, b] as an array literal', () => {
const result = parseEnvSpecDotEnvFile('VAL=[a, b]');
const valNode = result.configItems[0].value;
expectInstanceOf(valNode, ParsedEnvSpecArrayLiteral);
expect(valNode.simplifiedValue).toEqual(['a', 'b']);
});

it('parses quoted email strings in array literals', () => {
const result = parseEnvSpecDotEnvFile('VAL=["a@example.com", "b@example.com"]');
const valNode = result.configItems[0].value;
expectInstanceOf(valNode, ParsedEnvSpecArrayLiteral);
expect(valNode.simplifiedValue).toEqual(['a@example.com', 'b@example.com']);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,8 @@ Standalone objects and arrays use distinct brackets, keeping them unambiguous fr
- `[ value, ... ]` is an **array**
- they may be nested, and used as decorator values or as function-call arguments
- they are parsed using the common value-handling rules, and may span multiple lines (see [Multi-line literals](#multi-line-literals) below)
- they are **not yet supported as standalone config item values** (e.g. `ITEM={...}`) — for now an item value that starts with `{`/`[` is still treated as a string
- **array literals** are supported as standalone config item values (e.g. `ITEM=[a, b]`) when paired with `@type=array(...)`
- **object literals** are **not yet supported** as standalone config item values (e.g. `ITEM={...}`) — for now an item value that starts with `{` is still treated as a string

```env-spec
# @dec={ key1=v1, key2="v2", nested={ a=1 } }
Expand Down
30 changes: 30 additions & 0 deletions packages/varlock-website/src/content/docs/reference/data-types.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -254,4 +254,34 @@ POLL_INTERVAL=15m

Same parser is used by `cache(..., ttl=...)` and the plugin `cacheTtl` option, so any string that works there also works here. For cache mode behavior and troubleshooting, see the [Caching guide](/guides/caching/).
</div>

<div>
### `array`

Validates a list of values, coercing and checking each element with an inner type.

**Element type** — first positional argument: a type name (`string`, `email`, …) or a nested call for types with positional args (`enum(dev, staging, prod)`).

**Array options:**
- `minLength` / `maxLength` — element count bounds

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probable want an isLength too to match our string type

- `unique` (boolean) — reject duplicate elements
- `separator` (string) — split string input (e.g. `","` for comma-separated `.env` values)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both in terms of input and output, we may need some options around quotes and brackets.

For example, all these seem like reasonable inputs - a,b,c, [a,b,c], ["a", "b", "c"]
All of them could be parsed and the type-safe output will be fine, but the its not clear how we'd serialzie it back to a string for the basic env var. Also probably want to enable/disable basic splitting

Id assume default is that splitting by comma is fine, and no quotes needed, since if youre using an env var today to pass in a list, its probably just a,b,c

- `allowEmpty` (boolean) — allow `[]` (default: false)

Named options not listed above are forwarded to the element type (e.g. `normalize=true` on `array(email, …)`).

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is weird


```env-spec
# @type=array(email, normalize=true)
ALLOWED_EMAILS=[admin@example.com, support@example.com]

# @type=array(enum(dev, staging, prod))
APP_MODES=[dev, staging]

# Comma-separated in .env
# @type=array(email, separator=",")
ALLOWED_EMAILS=one@example.com, two@example.com
```

Array values are available as typed arrays via `ENV` (e.g. `string[]`). In `process.env` they are JSON-serialized.
</div>
</div>
7 changes: 6 additions & 1 deletion packages/varlock/src/cli/commands/run.command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { loadVarlockEnvGraph } from '../../lib/load-graph';
import { checkForConfigErrors, checkForNoEnvFiles, checkForSchemaErrors } from '../helpers/error-checks';
import { type TypedGunshiCommandFn } from '../helpers/gunshi-type-utils';
import { CliExitError } from '../helpers/exit-error';
import { serializeEnvValueForProcessEnv } from '../../lib/serialize-env-value';

export const commandSpec = define({
name: 'run',
Expand Down Expand Up @@ -203,6 +204,10 @@ export const commandFn: TypedGunshiCommandFn<typeof commandSpec> = async (ctx) =
// (e.g. a nested `varlock run` whose own resolution needs a secret-zero token)
const includeInternal = !!ctx.values['include-internal'];
const resolvedEnv = envGraph.getResolvedEnvObject({ includeInternal });
const serializedEnvForProcess: Record<string, string> = {};
for (const [key, value] of Object.entries(resolvedEnv)) {
serializedEnvForProcess[key] = serializeEnvValueForProcessEnv(value);
}
const serializedGraph = envGraph.getSerializedGraph();
const { resetRedactionMap } = await import('../../runtime/env');
const { createRedactedStreamWriter } = await import('../../runtime/lib/redact-stream');
Expand All @@ -224,7 +229,7 @@ export const commandFn: TypedGunshiCommandFn<typeof commandSpec> = async (ctx) =

const fullInjectedEnv: NodeJS.ProcessEnv = {
...process.env,
...(injectVars ? resolvedEnv : {}),
...(injectVars ? serializedEnvForProcess : {}),
__VARLOCK_RUN: '1', // flag for a child process to detect it is running via `varlock run`
...(injectBlob ? { __VARLOCK_ENV: JSON.stringify(serializedGraph) } : {}),
};
Expand Down
32 changes: 18 additions & 14 deletions packages/varlock/src/env-graph/lib/config-item.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,17 @@ import { EnvGraphDataSource } from './data-source';
import {
convertParsedValueToResolvers, type ResolvedValue, Resolver, StaticValueResolver,
} from './resolver';
import {
createDataTypeFromParsedType,
parseTypeDecoratorValue,
} from './type-decorator';
import { ItemDecoratorInstance } from './decorators';

export type ConfigItemDef = {
description?: string;
// TODO: translate parser decorator class into our own generic version
parsedDecorators?: Array<ParsedEnvSpecDecorator>;
parsedValue: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | undefined;
parsedValue: ParsedEnvSpecStaticValue | ParsedEnvSpecFunctionCall | ParsedEnvSpecArrayLiteral | undefined;

resolver?: Resolver;
decorators?: Array<ItemDecoratorInstance>;
Expand Down Expand Up @@ -321,30 +325,30 @@ export class ConfigItem {
}

const typeDec = this.getDec('type');
let dataTypeName: string | undefined;
let dataTypeArgs: any;
// TODO: this will not currently support any resolver functions within type settings
const typeDecParsedValue = typeDec?.parsedDecorator.value;
if (typeDecParsedValue instanceof ParsedEnvSpecStaticValue) {
dataTypeName = typeDecParsedValue.value;
} else if (typeDecParsedValue instanceof ParsedEnvSpecFunctionCall) {
dataTypeName = typeDecParsedValue.name;
dataTypeArgs = typeDecParsedValue.simplifiedArgs;
}
// if no type is set explicitly, we can try to use inferred type from the resolver
// currently only static value resolver does this - but you can imagine another resolver knowing the type ahead of time
// (maybe we only want to do this if the value is set in a schema file? or if all inferred types match?)
const parsedType = parseTypeDecoratorValue(typeDecParsedValue);
let dataTypeName = parsedType?.name;
if (!dataTypeName && this.valueResolver?.inferredType) {
dataTypeName = this.valueResolver.inferredType;
}
dataTypeName ||= 'string';
dataTypeArgs ||= [];

if (!(dataTypeName in this.envGraph.dataTypesRegistry)) {
this._schemaErrors.push(new SchemaError(`unknown data type: ${dataTypeName}`));
} else if (parsedType) {
try {
this.dataType = createDataTypeFromParsedType(this.envGraph.dataTypesRegistry, parsedType);
} catch (err) {
if (err instanceof SchemaError) {
this._schemaErrors.push(err);
} else {
throw err;
}
}
} else {
const dataTypeFactory = this.envGraph.dataTypesRegistry[dataTypeName];
this.dataType = dataTypeFactory(..._.isPlainObject(dataTypeArgs) ? [dataTypeArgs] : dataTypeArgs);
this.dataType = dataTypeFactory();
}
}

Expand Down
Loading
Loading