Skip to content

Commit 31db513

Browse files
committed
Fix multiprocessing table name collision
1 parent bdeedd7 commit 31db513

1 file changed

Lines changed: 5 additions & 2 deletions

File tree

python/tests/_pickle_multiprocessing_helpers.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,11 @@ def apply_builtin_expr(args: tuple) -> list:
7575
def apply_udf_expr(args: tuple) -> list:
7676
expr, values = args
7777
# Reuse the worker's global ctx so the UDF registered by the initializer
78-
# is visible during execution as well as during arg unpickling.
78+
# is visible during execution as well as during arg unpickling. Omit the
79+
# table name so each call gets a fresh auto-generated one — a worker may
80+
# process multiple tasks, and reusing a fixed name on the shared ctx would
81+
# collide on the second call.
7982
ctx = SessionContext.global_ctx()
8083
batch = pa.RecordBatch.from_arrays([pa.array(values, type=pa.int64())], names=["a"])
81-
df = ctx.create_dataframe([[batch]], name="t_udf")
84+
df = ctx.create_dataframe([[batch]])
8285
return df.select(expr.alias("out")).collect()[0].column(0).to_pylist()

0 commit comments

Comments
 (0)