Search.setIndex({"docnames": ["api", "api/dataframe", "api/dataset", "api/execution", "api/nodes", "api/tasks", "generated/smallpond.dataframe.DataFrame.compute", "generated/smallpond.dataframe.DataFrame.count", "generated/smallpond.dataframe.DataFrame.filter", "generated/smallpond.dataframe.DataFrame.flat_map", "generated/smallpond.dataframe.DataFrame.is_computed", "generated/smallpond.dataframe.DataFrame.limit", "generated/smallpond.dataframe.DataFrame.map", "generated/smallpond.dataframe.DataFrame.map_batches", "generated/smallpond.dataframe.DataFrame.partial_sort", "generated/smallpond.dataframe.DataFrame.random_shuffle", "generated/smallpond.dataframe.DataFrame.recompute", "generated/smallpond.dataframe.DataFrame.repartition", "generated/smallpond.dataframe.DataFrame.take", "generated/smallpond.dataframe.DataFrame.take_all", "generated/smallpond.dataframe.DataFrame.to_arrow", "generated/smallpond.dataframe.DataFrame.to_pandas", "generated/smallpond.dataframe.DataFrame.write_parquet", "generated/smallpond.dataframe.DataFrame.write_parquet_lazy", "generated/smallpond.dataframe.Session.from_arrow", "generated/smallpond.dataframe.Session.from_items", "generated/smallpond.dataframe.Session.from_pandas", "generated/smallpond.dataframe.Session.partial_sql", "generated/smallpond.dataframe.Session.read_csv", "generated/smallpond.dataframe.Session.read_json", "generated/smallpond.dataframe.Session.read_parquet", "generated/smallpond.dataframe.Session.wait", "generated/smallpond.execution.driver.Driver", "generated/smallpond.execution.executor.Executor", "generated/smallpond.execution.manager.JobManager", "generated/smallpond.execution.scheduler.Scheduler", "generated/smallpond.execution.task.ArrowBatchTask", "generated/smallpond.execution.task.ArrowComputeTask", "generated/smallpond.execution.task.ArrowStreamTask", "generated/smallpond.execution.task.DataSinkTask", "generated/smallpond.execution.task.DataSourceTask", "generated/smallpond.execution.task.EvenlyDistributedPartitionProducerTask", "generated/smallpond.execution.task.ExecutionPlan", "generated/smallpond.execution.task.HashPartitionArrowTask", "generated/smallpond.execution.task.HashPartitionDuckDbTask", "generated/smallpond.execution.task.HashPartitionTask", "generated/smallpond.execution.task.JobId", "generated/smallpond.execution.task.LoadPartitionedDataSetProducerTask", "generated/smallpond.execution.task.MergeDataSetsTask", "generated/smallpond.execution.task.PandasBatchTask", "generated/smallpond.execution.task.PandasComputeTask", "generated/smallpond.execution.task.PartitionConsumerTask", "generated/smallpond.execution.task.PartitionInfo", "generated/smallpond.execution.task.PartitionProducerTask", "generated/smallpond.execution.task.PerfStats", "generated/smallpond.execution.task.ProjectionTask", "generated/smallpond.execution.task.PythonScriptTask", "generated/smallpond.execution.task.RangePartitionTask", "generated/smallpond.execution.task.RepeatPartitionProducerTask", "generated/smallpond.execution.task.RootTask", "generated/smallpond.execution.task.RuntimeContext", "generated/smallpond.execution.task.SplitDataSetTask", "generated/smallpond.execution.task.SqlEngineTask", "generated/smallpond.execution.task.Task", "generated/smallpond.execution.task.TaskId", "generated/smallpond.execution.task.TaskRuntimeId", "generated/smallpond.execution.task.UserDefinedPartitionProducerTask", "generated/smallpond.init", "generated/smallpond.logical.dataset.ArrowTableDataSet", "generated/smallpond.logical.dataset.CsvDataSet", "generated/smallpond.logical.dataset.DataSet", "generated/smallpond.logical.dataset.FileSet", "generated/smallpond.logical.dataset.JsonDataSet", "generated/smallpond.logical.dataset.PandasDataSet", "generated/smallpond.logical.dataset.ParquetDataSet", "generated/smallpond.logical.dataset.PartitionedDataSet", "generated/smallpond.logical.dataset.SqlQueryDataSet", "generated/smallpond.logical.node.ArrowBatchNode", "generated/smallpond.logical.node.ArrowComputeNode", "generated/smallpond.logical.node.ArrowStreamNode", "generated/smallpond.logical.node.ConsolidateNode", "generated/smallpond.logical.node.Context", "generated/smallpond.logical.node.DataSetPartitionNode", "generated/smallpond.logical.node.DataSinkNode", "generated/smallpond.logical.node.DataSourceNode", "generated/smallpond.logical.node.EvenlyDistributedPartitionNode", "generated/smallpond.logical.node.HashPartitionNode", "generated/smallpond.logical.node.LimitNode", "generated/smallpond.logical.node.LoadPartitionedDataSetNode", "generated/smallpond.logical.node.LogicalPlan", "generated/smallpond.logical.node.LogicalPlanVisitor", "generated/smallpond.logical.node.Node", "generated/smallpond.logical.node.NodeId", "generated/smallpond.logical.node.PandasBatchNode", "generated/smallpond.logical.node.PandasComputeNode", "generated/smallpond.logical.node.PartitionNode", "generated/smallpond.logical.node.ProjectionNode", "generated/smallpond.logical.node.PythonScriptNode", "generated/smallpond.logical.node.RangePartitionNode", "generated/smallpond.logical.node.RepeatPartitionNode", "generated/smallpond.logical.node.RootNode", "generated/smallpond.logical.node.ShuffleNode", "generated/smallpond.logical.node.SqlEngineNode", "generated/smallpond.logical.node.UnionNode", "generated/smallpond.logical.node.UserDefinedPartitionNode", "generated/smallpond.logical.node.UserPartitionedDataSourceNode", "generated/smallpond.platform.MPI", "generated/smallpond.platform.Platform", "getstarted", "index", "internals"], "filenames": ["api.rst", "api/dataframe.rst", "api/dataset.rst", "api/execution.rst", "api/nodes.rst", "api/tasks.rst", "generated/smallpond.dataframe.DataFrame.compute.rst", "generated/smallpond.dataframe.DataFrame.count.rst", "generated/smallpond.dataframe.DataFrame.filter.rst", "generated/smallpond.dataframe.DataFrame.flat_map.rst", "generated/smallpond.dataframe.DataFrame.is_computed.rst", "generated/smallpond.dataframe.DataFrame.limit.rst", "generated/smallpond.dataframe.DataFrame.map.rst", "generated/smallpond.dataframe.DataFrame.map_batches.rst", "generated/smallpond.dataframe.DataFrame.partial_sort.rst", "generated/smallpond.dataframe.DataFrame.random_shuffle.rst", "generated/smallpond.dataframe.DataFrame.recompute.rst", "generated/smallpond.dataframe.DataFrame.repartition.rst", "generated/smallpond.dataframe.DataFrame.take.rst", "generated/smallpond.dataframe.DataFrame.take_all.rst", "generated/smallpond.dataframe.DataFrame.to_arrow.rst", "generated/smallpond.dataframe.DataFrame.to_pandas.rst", "generated/smallpond.dataframe.DataFrame.write_parquet.rst", "generated/smallpond.dataframe.DataFrame.write_parquet_lazy.rst", "generated/smallpond.dataframe.Session.from_arrow.rst", "generated/smallpond.dataframe.Session.from_items.rst", "generated/smallpond.dataframe.Session.from_pandas.rst", "generated/smallpond.dataframe.Session.partial_sql.rst", "generated/smallpond.dataframe.Session.read_csv.rst", "generated/smallpond.dataframe.Session.read_json.rst", "generated/smallpond.dataframe.Session.read_parquet.rst", "generated/smallpond.dataframe.Session.wait.rst", "generated/smallpond.execution.driver.Driver.rst", "generated/smallpond.execution.executor.Executor.rst", "generated/smallpond.execution.manager.JobManager.rst", "generated/smallpond.execution.scheduler.Scheduler.rst", "generated/smallpond.execution.task.ArrowBatchTask.rst", "generated/smallpond.execution.task.ArrowComputeTask.rst", "generated/smallpond.execution.task.ArrowStreamTask.rst", "generated/smallpond.execution.task.DataSinkTask.rst", "generated/smallpond.execution.task.DataSourceTask.rst", "generated/smallpond.execution.task.EvenlyDistributedPartitionProducerTask.rst", "generated/smallpond.execution.task.ExecutionPlan.rst", "generated/smallpond.execution.task.HashPartitionArrowTask.rst", "generated/smallpond.execution.task.HashPartitionDuckDbTask.rst", "generated/smallpond.execution.task.HashPartitionTask.rst", "generated/smallpond.execution.task.JobId.rst", "generated/smallpond.execution.task.LoadPartitionedDataSetProducerTask.rst", "generated/smallpond.execution.task.MergeDataSetsTask.rst", "generated/smallpond.execution.task.PandasBatchTask.rst", "generated/smallpond.execution.task.PandasComputeTask.rst", "generated/smallpond.execution.task.PartitionConsumerTask.rst", "generated/smallpond.execution.task.PartitionInfo.rst", "generated/smallpond.execution.task.PartitionProducerTask.rst", "generated/smallpond.execution.task.PerfStats.rst", "generated/smallpond.execution.task.ProjectionTask.rst", "generated/smallpond.execution.task.PythonScriptTask.rst", "generated/smallpond.execution.task.RangePartitionTask.rst", "generated/smallpond.execution.task.RepeatPartitionProducerTask.rst", "generated/smallpond.execution.task.RootTask.rst", "generated/smallpond.execution.task.RuntimeContext.rst", "generated/smallpond.execution.task.SplitDataSetTask.rst", "generated/smallpond.execution.task.SqlEngineTask.rst", "generated/smallpond.execution.task.Task.rst", "generated/smallpond.execution.task.TaskId.rst", "generated/smallpond.execution.task.TaskRuntimeId.rst", "generated/smallpond.execution.task.UserDefinedPartitionProducerTask.rst", "generated/smallpond.init.rst", "generated/smallpond.logical.dataset.ArrowTableDataSet.rst", "generated/smallpond.logical.dataset.CsvDataSet.rst", "generated/smallpond.logical.dataset.DataSet.rst", "generated/smallpond.logical.dataset.FileSet.rst", "generated/smallpond.logical.dataset.JsonDataSet.rst", "generated/smallpond.logical.dataset.PandasDataSet.rst", "generated/smallpond.logical.dataset.ParquetDataSet.rst", "generated/smallpond.logical.dataset.PartitionedDataSet.rst", "generated/smallpond.logical.dataset.SqlQueryDataSet.rst", "generated/smallpond.logical.node.ArrowBatchNode.rst", "generated/smallpond.logical.node.ArrowComputeNode.rst", "generated/smallpond.logical.node.ArrowStreamNode.rst", "generated/smallpond.logical.node.ConsolidateNode.rst", "generated/smallpond.logical.node.Context.rst", "generated/smallpond.logical.node.DataSetPartitionNode.rst", "generated/smallpond.logical.node.DataSinkNode.rst", "generated/smallpond.logical.node.DataSourceNode.rst", "generated/smallpond.logical.node.EvenlyDistributedPartitionNode.rst", "generated/smallpond.logical.node.HashPartitionNode.rst", "generated/smallpond.logical.node.LimitNode.rst", "generated/smallpond.logical.node.LoadPartitionedDataSetNode.rst", "generated/smallpond.logical.node.LogicalPlan.rst", "generated/smallpond.logical.node.LogicalPlanVisitor.rst", "generated/smallpond.logical.node.Node.rst", "generated/smallpond.logical.node.NodeId.rst", "generated/smallpond.logical.node.PandasBatchNode.rst", "generated/smallpond.logical.node.PandasComputeNode.rst", "generated/smallpond.logical.node.PartitionNode.rst", "generated/smallpond.logical.node.ProjectionNode.rst", "generated/smallpond.logical.node.PythonScriptNode.rst", "generated/smallpond.logical.node.RangePartitionNode.rst", "generated/smallpond.logical.node.RepeatPartitionNode.rst", "generated/smallpond.logical.node.RootNode.rst", "generated/smallpond.logical.node.ShuffleNode.rst", "generated/smallpond.logical.node.SqlEngineNode.rst", "generated/smallpond.logical.node.UnionNode.rst", "generated/smallpond.logical.node.UserDefinedPartitionNode.rst", "generated/smallpond.logical.node.UserPartitionedDataSourceNode.rst", "generated/smallpond.platform.MPI.rst", "generated/smallpond.platform.Platform.rst", "getstarted.rst", "index.rst", "internals.rst"], "titles": ["API Reference", "DataFrame", "Dataset", "Execution", "Nodes", "Tasks", "smallpond.dataframe.DataFrame.compute", "smallpond.dataframe.DataFrame.count", "smallpond.dataframe.DataFrame.filter", "smallpond.dataframe.DataFrame.flat_map", "smallpond.dataframe.DataFrame.is_computed", "smallpond.dataframe.DataFrame.limit", "smallpond.dataframe.DataFrame.map", "smallpond.dataframe.DataFrame.map_batches", "smallpond.dataframe.DataFrame.partial_sort", "smallpond.dataframe.DataFrame.random_shuffle", "smallpond.dataframe.DataFrame.recompute", "smallpond.dataframe.DataFrame.repartition", "smallpond.dataframe.DataFrame.take", "smallpond.dataframe.DataFrame.take_all", "smallpond.dataframe.DataFrame.to_arrow", "smallpond.dataframe.DataFrame.to_pandas", "smallpond.dataframe.DataFrame.write_parquet", "smallpond.dataframe.DataFrame.write_parquet_lazy", "smallpond.dataframe.Session.from_arrow", "smallpond.dataframe.Session.from_items", "smallpond.dataframe.Session.from_pandas", "smallpond.dataframe.Session.partial_sql", "smallpond.dataframe.Session.read_csv", "smallpond.dataframe.Session.read_json", "smallpond.dataframe.Session.read_parquet", "smallpond.dataframe.Session.wait", "smallpond.execution.driver.Driver", "smallpond.execution.executor.Executor", "smallpond.execution.manager.JobManager", "smallpond.execution.scheduler.Scheduler", "smallpond.execution.task.ArrowBatchTask", "smallpond.execution.task.ArrowComputeTask", "smallpond.execution.task.ArrowStreamTask", "smallpond.execution.task.DataSinkTask", "smallpond.execution.task.DataSourceTask", "smallpond.execution.task.EvenlyDistributedPartitionProducerTask", "smallpond.execution.task.ExecutionPlan", "smallpond.execution.task.HashPartitionArrowTask", "smallpond.execution.task.HashPartitionDuckDbTask", "smallpond.execution.task.HashPartitionTask", "smallpond.execution.task.JobId", "smallpond.execution.task.LoadPartitionedDataSetProducerTask", "smallpond.execution.task.MergeDataSetsTask", "smallpond.execution.task.PandasBatchTask", "smallpond.execution.task.PandasComputeTask", "smallpond.execution.task.PartitionConsumerTask", "smallpond.execution.task.PartitionInfo", "smallpond.execution.task.PartitionProducerTask", "smallpond.execution.task.PerfStats", "smallpond.execution.task.ProjectionTask", "smallpond.execution.task.PythonScriptTask", "smallpond.execution.task.RangePartitionTask", "smallpond.execution.task.RepeatPartitionProducerTask", "smallpond.execution.task.RootTask", "smallpond.execution.task.RuntimeContext", "smallpond.execution.task.SplitDataSetTask", "smallpond.execution.task.SqlEngineTask", "smallpond.execution.task.Task", "smallpond.execution.task.TaskId", "smallpond.execution.task.TaskRuntimeId", "smallpond.execution.task.UserDefinedPartitionProducerTask", "smallpond.init", "smallpond.logical.dataset.ArrowTableDataSet", "smallpond.logical.dataset.CsvDataSet", "smallpond.logical.dataset.DataSet", "smallpond.logical.dataset.FileSet", "smallpond.logical.dataset.JsonDataSet", "smallpond.logical.dataset.PandasDataSet", "smallpond.logical.dataset.ParquetDataSet", "smallpond.logical.dataset.PartitionedDataSet", "smallpond.logical.dataset.SqlQueryDataSet", "smallpond.logical.node.ArrowBatchNode", "smallpond.logical.node.ArrowComputeNode", "smallpond.logical.node.ArrowStreamNode", "smallpond.logical.node.ConsolidateNode", "smallpond.logical.node.Context", "smallpond.logical.node.DataSetPartitionNode", "smallpond.logical.node.DataSinkNode", "smallpond.logical.node.DataSourceNode", "smallpond.logical.node.EvenlyDistributedPartitionNode", "smallpond.logical.node.HashPartitionNode", "smallpond.logical.node.LimitNode", "smallpond.logical.node.LoadPartitionedDataSetNode", "smallpond.logical.node.LogicalPlan", "smallpond.logical.node.LogicalPlanVisitor", "smallpond.logical.node.Node", "smallpond.logical.node.NodeId", "smallpond.logical.node.PandasBatchNode", "smallpond.logical.node.PandasComputeNode", "smallpond.logical.node.PartitionNode", "smallpond.logical.node.ProjectionNode", "smallpond.logical.node.PythonScriptNode", "smallpond.logical.node.RangePartitionNode", "smallpond.logical.node.RepeatPartitionNode", "smallpond.logical.node.RootNode", "smallpond.logical.node.ShuffleNode", "smallpond.logical.node.SqlEngineNode", "smallpond.logical.node.UnionNode", "smallpond.logical.node.UserDefinedPartitionNode", "smallpond.logical.node.UserPartitionedDataSourceNode", "smallpond.platform.MPI", "smallpond.platform.Platform", "Getting Started", "smallpond", "Internals"], "terms": {"smallpond": [0, 1, 3, 108, 110], "provid": [0, 7, 27, 77, 78, 79, 86, 93, 94, 101, 102], "both": [0, 91], "current": [0, 3, 67, 102], "two": [0, 27, 103], "differ": [0, 68, 69, 70, 71, 72, 73, 74, 75, 76, 96], "support": [0, 3, 88, 96, 102, 110], "dynam": 0, "static": 0, "construct": [0, 3, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 83, 84, 86, 87, 93, 94, 96, 97, 101, 102], "data": [0, 4, 6, 10, 13, 16, 17, 22, 23, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 84, 86, 93, 94, 96, 101, 102, 103, 109], "flow": [0, 103], "graph": [0, 4, 42, 63, 89, 110], "respect": [0, 3], "due": 0, "histor": 0, "reason": 0, "us": [0, 1, 3, 12, 31, 67, 77, 78, 79, 81, 82, 86, 91, 93, 94, 97, 101, 102, 108, 109, 110], "schedul": [0, 5, 63, 77, 78, 79, 83, 91, 93, 94, 108, 110], "backend": 0, "configur": [0, 3, 60, 110], "option": [0, 9, 12, 13, 17, 46, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 83, 85, 86, 88, 91, 93, 94, 95, 96, 97, 98, 99, 101, 102, 104, 105], "The": [0, 3, 4, 9, 12, 13, 16, 17, 27, 33, 35, 46, 60, 63, 67, 70, 76, 77, 78, 79, 82, 83, 86, 87, 88, 89, 91, 93, 94, 95, 96, 98, 99, 101, 102, 104, 105, 108], "rai": [0, 67, 108], "execut": [0, 5, 6, 7, 11, 18, 19, 20, 21, 27, 102, 108, 109, 110], "built": [0, 3], "onli": [0, 3, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 88, 91, 93, 94, 96, 101, 102], "one": [0, 3, 4, 46, 102, 103], "time": [0, 63, 67], "howev": [0, 3], "offer": 0, "more": [0, 7, 8, 9, 12, 67, 77, 78, 79, 86, 93, 94, 101, 102, 103, 108], "perform": [0, 1, 6, 7, 18, 19, 20, 21, 54, 83, 91, 102, 109], "optim": 0, "richer": 0, "we": [0, 77, 78, 79, 86, 93, 94, 101, 102], "ar": [0, 1, 3, 4, 27, 46, 67, 84, 86, 88, 91, 95, 96, 98, 99, 101, 102, 104, 105], "work": [0, 63], "merg": 0, "them": [0, 3], "so": 0, "futur": 0, "you": [0, 1, 3, 4, 5, 27, 108], "can": [0, 1, 3, 4, 5, 7, 16, 27, 31, 63, 83, 91, 102, 108, 110], "unifi": [0, 68, 69, 70, 71, 72, 73, 74, 75, 76, 96], "freeli": 0, "choos": [0, 67], "between": [0, 110], "i": [0, 1, 3, 4, 8, 9, 10, 12, 22, 23, 27, 46, 63, 67, 75, 77, 78, 79, 82, 83, 88, 91, 93, 94, 95, 96, 98, 99, 102, 104, 105, 108, 109, 110], "center": 0, "around": [0, 3], "datafram": [0, 73, 93, 94, 108, 109], "It": [0, 1, 3, 13, 46, 67, 84, 109], "allow": [0, 83, 91], "result": [0, 9, 16, 46, 76, 102], "retriev": 0, "A": [0, 1, 3, 4, 8, 9, 12, 13, 14, 32, 42, 46, 64, 65, 68, 69, 70, 71, 72, 73, 74, 75, 76, 84, 92, 100, 102, 110], "typic": [0, 1, 4], "workflow": [0, 1, 4], "look": [0, 1], "like": [0, 1], "thi": [0, 1, 6, 7, 11, 18, 19, 20, 21, 22, 23, 31, 67, 77, 78, 79, 86, 88, 91, 93, 94, 95, 96, 98, 99, 101, 104, 105, 110], "import": [0, 1, 3, 67, 102, 108], "sp": [0, 1, 23, 27, 31, 67, 108], "init": [0, 1, 3, 108], "df": [0, 1, 8, 9, 12, 14, 16, 17, 22, 23, 26, 31, 73, 108], "read_parquet": [0, 1, 27, 31, 108], "path": [0, 1, 2, 3, 4, 12, 22, 23, 28, 29, 30, 68, 69, 70, 71, 72, 73, 74, 75, 76, 82, 83, 85, 91, 102, 108], "dataset": [0, 1, 4, 17, 27, 40, 52, 56, 63, 66, 77, 78, 79, 82, 84, 85, 88, 93, 94, 95, 97, 98, 99, 102, 104, 105, 108, 109, 110], "parquet": [0, 1, 2, 4, 22, 23, 27, 30, 31, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 85, 86, 88, 91, 93, 94, 101, 102, 108, 109], "repartit": [0, 1, 27, 108], "10": [0, 1, 3, 17, 27, 67, 88, 95, 98, 99, 104, 105], "map": [0, 1, 9, 31, 108], "x": [0, 1], "1": [0, 1, 8, 27, 48, 52, 60, 62, 63, 67, 77, 78, 79, 85, 88, 91, 93, 94, 95, 97, 98, 99, 102, 104, 105, 108], "write_parquet": [0, 1, 23, 31, 67, 108], "output": [0, 1, 4, 9, 12, 22, 31, 63, 67, 82, 83, 85, 86, 87, 88, 91, 95, 96, 98, 99, 100, 101, 102, 104, 105, 108, 110], "initi": [0, 5, 63, 67], "load": [0, 12, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 88, 93, 94, 96, 102], "partit": [0, 4, 14, 17, 27, 52, 75, 80, 82, 85, 86, 88, 95, 98, 99, 101, 102, 103, 104, 105], "transform": [0, 4, 6, 7, 18, 19, 20, 21], "consum": [0, 108], "recommend": 0, "In": [0, 91], "user": [0, 3, 12, 67, 77, 78, 79, 91, 93, 94, 97, 102, 104, 108], "manual": [0, 1, 108], "creat": [0, 2, 3, 4, 5, 24, 25, 26, 28, 29, 30, 46, 67, 77, 78, 79, 83, 84, 86, 88, 91, 93, 95, 96, 97, 98, 99, 101, 104, 105, 108], "node": [0, 46, 63, 67, 109], "submit": 0, "gener": [0, 3, 4, 46, 67, 77, 78, 79, 86, 91, 93, 94, 96, 101, 102], "task": [0, 3, 4, 33, 35, 67, 77, 78, 79, 91, 93, 94, 97, 108, 109, 110], "wait": [0, 23], "all": [0, 15, 19, 31, 46, 63, 67, 70, 84, 91, 95, 96, 102, 103, 107, 110], "complet": [0, 110], "exampl": [0, 46, 77, 78, 79, 93, 94, 97, 102], "shown": 0, "below": 0, "from": [0, 3, 4, 7, 9, 12, 24, 25, 26, 27, 28, 29, 30, 31, 46, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 91, 93, 94, 96, 101, 102, 108, 110], "logic": [0, 3, 4, 5, 32], "parquetdataset": [0, 2, 4], "context": [0, 5, 17, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "datasourcenod": [0, 4], "datasetpartitionnod": [0, 4], "sqlenginenod": [0, 4], "logicalplan": [0, 3, 42], "driver": [0, 3], "def": [0, 3, 12, 96], "my_pipelin": [0, 3], "input_path": [0, 3], "list": [0, 3, 9, 12, 14, 17, 18, 19, 25, 28, 29, 30, 36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 85, 86, 93, 94, 96, 97, 98, 102, 105], "str": [0, 3, 8, 9, 12, 14, 17, 18, 19, 22, 23, 27, 28, 29, 30, 33, 34, 36, 37, 38, 39, 41, 43, 44, 45, 47, 48, 49, 50, 52, 53, 55, 56, 58, 60, 62, 63, 66, 67, 69, 70, 71, 72, 74, 76, 77, 78, 79, 80, 83, 85, 86, 88, 91, 93, 94, 95, 96, 97, 98, 99, 101, 102, 104, 105], "npartit": [0, 3, 4, 17, 41, 43, 44, 45, 47, 52, 53, 58, 66, 82, 85, 86, 88, 95, 98, 99, 101, 104, 105], "int": [0, 3, 7, 11, 12, 13, 17, 18, 35, 36, 37, 38, 41, 43, 44, 45, 46, 47, 48, 49, 50, 52, 53, 56, 58, 60, 62, 63, 65, 66, 67, 69, 77, 78, 79, 82, 85, 86, 87, 88, 91, 93, 94, 95, 97, 98, 99, 101, 102, 104], "ctx": [0, 4, 33, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "select": [0, 3, 4, 16, 27, 77, 78, 79, 86, 93, 94, 96, 101, 102], "0": [0, 4, 27, 48, 52, 60, 62, 63, 67, 77, 78, 79, 91, 93, 94, 96, 97, 102, 108], "return": [0, 1, 4, 8, 9, 12, 13, 18, 19, 78, 79, 96, 97, 102], "__name__": [0, 3], "__main__": [0, 3], "add_argu": [0, 3], "narg": [0, 3], "n": [0, 3, 82, 85, 86, 88, 95, 98, 99, 104, 105], "type": [0, 3, 39, 83], "default": [0, 3, 13, 17, 34, 67, 82, 83, 85, 91, 102], "plan": [0, 3, 4, 5, 32, 63, 81, 84, 89, 90, 91], "get_argu": [0, 3], "run": [0, 3, 60, 63, 67, 77, 78, 79, 93, 94, 97, 102, 109], "To": [0, 2, 67, 108], "script": [0, 3, 67], "python": [0, 3, 12, 25, 77, 78, 79, 81, 86, 93, 94, 97, 101, 108], "py": [0, 3, 77, 78, 79, 82, 88, 93, 94, 95, 97, 98, 99, 104, 105], "runtimecontext": [0, 33, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66, 77, 78, 79, 93, 94, 96, 97], "executionplan": [0, 35], "job": [0, 46, 60], "executor": [0, 67, 83, 91], "custom": [0, 83, 91], "platform": [0, 34, 67], "main": 1, "class": [1, 3, 13, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107], "repres": [1, 2, 4, 63, 84], "lazili": 1, "comput": [1, 16, 31, 86, 88, 89, 95, 98, 99, 101, 104, 105, 109], "set": [1, 46, 67, 69, 71, 72, 74, 83, 84, 91, 96, 102, 103, 108], "appli": [1, 4, 9, 12, 13, 108], "new": [1, 4, 5, 99], "These": 1, "oper": [1, 4, 6, 7, 18, 19, 20, 21, 22, 23, 63, 83], "trigger": [1, 6, 7, 11, 18, 19, 20, 21], "lazi": [1, 6, 7, 18, 19, 20, 21], "method": [1, 3, 11, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107], "collect": [2, 83], "file": [2, 17, 22, 23, 28, 29, 30, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 83, 85, 86, 88, 91, 93, 94, 96, 101, 102, 104, 108, 109], "after": [3, 67], "jobmanag": 3, "cluster": [3, 67], "most": [3, 67], "case": 3, "need": [3, 27, 78, 79, 97], "entri": [3, 67], "point": [3, 67, 83], "entir": 3, "simpl": 3, "wrapper": 3, "read": [3, 7, 67, 77, 79, 93], "command": 3, "line": [3, 83], "argument": [3, 46, 67, 77, 78, 79, 93, 94, 97], "pass": [3, 9, 12, 102], "add": 3, "your": 3, "own": 3, "build": [3, 4], "lower": 3, "level": [3, 109, 110], "api": [3, 109], "thei": [3, 27, 63], "directli": 3, "respons": 3, "do": 3, "defin": [3, 12, 77, 78, 79, 89, 91, 93, 94, 97, 98, 102], "includ": [3, 32, 65, 91, 102], "seri": [3, 4, 22, 23, 63, 77, 93], "By": [3, 67, 102], "automat": [3, 67, 91], "detect": [3, 67], "environ": [3, 67], "suitabl": [3, 67], "If": [3, 7, 9, 12, 17, 27, 67, 77, 78, 79, 83, 91, 93, 94, 97, 102], "cannot": 3, "specifi": [3, 17, 63, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 86, 91, 93, 94, 101, 102, 108], "via": [3, 46], "paramet": 3, "mpi": [3, 67], "Or": [3, 78, 79, 97], "implement": [3, 78, 79, 97, 98], "my": 3, "myplatform": 3, "start_job": 3, "self": [3, 77, 78, 79, 93, 94, 96, 97], "sp_platform": [3, 67], "fundament": 4, "block": [4, 22, 23], "process": [4, 67, 77, 78, 79, 93, 94, 96, 97, 109], "pipelin": [4, 31], "each": [4, 9, 12, 13, 14, 27, 63, 77, 78, 79, 81, 86, 88, 92, 93, 94, 95, 98, 99, 101, 102, 104, 105], "encapsul": 4, "specif": [4, 63, 102], "chain": 4, "togeth": 4, "form": [4, 63], "which": [4, 67, 77, 78, 79, 93, 94, 97, 102], "direct": [4, 42, 63, 89], "acycl": [4, 42, 63, 89], "dag": [4, 42, 63], "overal": 4, "follow": [4, 16, 102, 110], "global": [4, 15, 81], "sourc": [4, 84], "2": [4, 96, 102], "sql": [4, 8, 9, 12, 27, 76, 102, 108], "engin": [4, 86, 101, 102, 109], "root": [4, 67, 83, 100], "see": [4, 5, 22, 23, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 83, 84, 86, 88, 93, 94, 95, 96, 97, 98, 99, 101, 102, 104, 105], "notabl": 4, "properti": 4, "input": [4, 8, 9, 12, 13, 16, 27, 31, 63, 77, 78, 79, 82, 84, 85, 87, 91, 93, 94, 96, 97, 102, 103], "mai": [4, 83], "write": [4, 22, 23, 77, 79, 86, 93, 101, 102], "share": [4, 83, 91], "storag": [4, 109], "simpli": 4, "recombin": 4, "runtim": [5, 65, 67, 78, 79, 97, 102], "runtime_ctx": [5, 78, 79, 96, 97, 110], "jobid": [5, 60], "data_root": [5, 34, 60, 67, 110], "socket": 5, "gethostnam": 5, "cleanup_root": 5, "true": [5, 17, 36, 38, 49, 62, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 82, 85, 86, 88, 93, 95, 98, 99, 101, 102, 104, 105, 108], "create_logical_plan": 5, "an": [5, 20, 46, 63, 67, 68, 78, 79, 83, 87, 96, 97], "planner": 5, "exec_plan": [5, 35, 110], "create_exec_plan": 5, "none": [6, 9, 12, 17, 22, 30, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 55, 56, 57, 58, 59, 60, 61, 62, 63, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 81, 82, 83, 84, 85, 86, 87, 88, 89, 91, 93, 94, 95, 96, 97, 98, 99, 101, 102, 104, 105], "number": [7, 11, 13, 17, 27, 67, 77, 78, 79, 80, 82, 86, 87, 91, 93, 94, 101, 102], "row": [7, 8, 9, 11, 12, 13, 14, 15, 17, 18, 19, 77, 78, 79, 82, 85, 86, 87, 93, 94, 101, 102, 104, 108], "consist": 7, "than": 7, "t": [7, 8, 11], "determin": 7, "metadata": 7, "datasourc": 7, "sql_or_func": [8, 9, 12], "callabl": [8, 9, 12, 13, 36, 37, 38, 49, 50, 56, 66, 76, 77, 78, 79, 93, 94, 97], "dict": [8, 9, 12, 18, 19, 28, 29, 60, 69, 72], "ani": [8, 9, 12, 18, 19, 25, 83, 84, 91], "bool": [8, 10, 17, 30, 43, 44, 45, 47, 55, 56, 67, 85, 86, 88, 95, 97, 98, 101, 102, 104], "kwarg": [8, 9, 12, 13, 14, 15, 17, 27, 67], "out": 8, "don": 8, "satisfi": 8, "given": [8, 11, 13, 14, 17, 22, 23, 46, 67], "predic": 8, "express": [8, 9, 12, 46, 91, 108], "function": [8, 9, 12, 13, 77, 78, 79, 93, 94, 97, 102, 108], "For": [8, 9, 12, 46, 88, 95, 98, 99, 102, 104, 105], "should": [8, 9, 12, 13, 63, 67, 77, 78, 79, 86, 91, 93, 94, 97, 101, 102, 103], "take": [8, 9, 11, 12, 13, 63], "dictionari": [8, 9, 12, 77, 78, 79, 86, 93, 94, 101, 102], "column": [8, 9, 12, 14, 17, 30, 55, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 82, 86, 93, 94, 96, 101, 102, 108], "boolean": 8, "prefer": [8, 9, 12], "": [8, 9, 12, 16, 81], "effici": [8, 9, 12, 109], "lambda": [8, 9, 12, 108], "r": 8, "schema": [9, 12, 28, 29, 69, 72], "flatten": 9, "infer": [9, 12], "first": [9, 12, 90, 96, 108], "valu": [9, 12, 67, 86, 96, 101], "unnest": 9, "arrai": 9, "b": [9, 12, 14, 16, 27, 108], "c": [9, 12, 27, 108], "check": [10, 96], "readi": 10, "disk": 10, "unlik": 11, "doesn": 11, "udf": [12, 62, 81, 102], "referenc": [12, 102], "param": 12, "udftyp": 12, "return_typ": 12, "gcd": 12, "while": 12, "duckdb": [12, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 93, 94, 96, 101, 102, 109], "extens": [12, 102], "duckdb_extens": [12, 102], "func": 13, "tabl": [13, 20, 24, 36, 37, 38, 49, 50, 68, 77, 78, 79, 86, 93, 94, 96, 101], "batch_siz": 13, "122880": [13, 36, 37, 38, 43, 44, 45, 49, 50, 62], "batch": [13, 77, 79, 93, 102, 110], "arrow": [13, 20, 68, 77, 78, 79, 86, 93, 94, 96, 101], "sort": 14, "desc": 14, "randomli": 15, "shuffl": [15, 80, 82, 85, 86, 101], "alwai": [16, 99], "regardless": 16, "whether": [16, 46], "alreadi": 16, "modifi": 16, "code": [16, 32, 77, 78, 79, 93, 94, 97, 104], "rerun": 16, "reus": 16, "hash_bi": [17, 27, 108], "by_row": [17, 108], "fals": [17, 30, 35, 36, 37, 38, 39, 41, 43, 44, 45, 49, 50, 60, 62, 69, 70, 71, 72, 74, 77, 78, 79, 82, 83, 85, 86, 88, 93, 94, 95, 98, 101, 102, 104], "would": [17, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 86, 87, 88, 91, 93, 94, 95, 96, 98, 99, 101, 104, 105], "split": [17, 77, 79, 88, 93, 95, 96, 98, 99, 102, 104, 105], "distribut": [17, 82, 85, 88, 95, 98, 99, 104, 105, 109], "size": [17, 77, 78, 79, 86, 93, 94, 101, 102], "hash": [17, 86, 101, 108], "instead": [17, 77, 78, 79, 82, 85, 93, 94], "evenli": [17, 82, 85], "host": [17, 96, 108], "bucket": 17, "limit": [18, 63, 77, 78, 79, 87, 91, 93, 94, 102], "up": 18, "convert": [20, 21, 86, 101], "panda": [21, 26, 73, 93, 94], "under": [22, 23, 68, 69, 70, 71, 72, 73, 74, 75, 76, 83], "write_parquet_lazi": 22, "non": [22, 23], "version": [22, 23, 46], "o1": 23, "output1": [23, 31], "o2": 23, "output2": [23, 31], "pyarrow": [24, 77, 78, 79, 86, 93, 94, 101], "item": [25, 102], "local": [25, 67, 108], "object": [25, 63], "queri": [27, 76, 86, 101, 102], "contain": 27, "placehold": [27, 102], "etc": [27, 91, 102], "multipl": [27, 31, 75, 100, 102], "must": [27, 46], "have": [27, 46, 77, 78, 79, 93, 94, 97, 102, 103], "same": [27, 46, 77, 78, 79, 83, 93, 94, 97, 103], "join": 27, "make": [27, 77, 78, 79, 86, 93, 94, 101, 102, 109], "sure": 27, "kei": [27, 82, 86, 101], "correctli": 27, "id": [27, 33, 65, 67, 102], "delim": [28, 69], "csv": [28, 69], "json": [29, 72], "recurs": [30, 68, 69, 70, 71, 72, 73, 74, 75, 76], "union_by_nam": [30, 55, 68, 69, 70, 71, 72, 73, 74, 75, 76, 96], "col1": 31, "col2": 31, "helper": 32, "boilerpl": 32, "__init__": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107], "attribut": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 83, 84, 85, 86, 87, 88, 89, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "wq": 33, "workqueu": 33, "cq": 33, "python_venv": 34, "task_imag": 34, "max_retry_count": 35, "5": [35, 60, 88, 95, 98, 99, 104, 105], "max_fail_count": 35, "3": [35, 36, 37, 38, 43, 44, 45, 46, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102, 108], "prioritize_retri": 35, "speculative_exec": 35, "liter": [35, 39, 83, 86, 96, 101], "disabl": [35, 102], "enabl": [35, 102, 109], "aggress": 35, "stop_executor_on_failur": 35, "nonzero_exitcode_as_oom": 35, "remove_output_root": 35, "sched_state_observ": 35, "input_dep": [36, 37, 38, 39, 41, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66, 77, 78, 79, 80, 82, 83, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "partition_info": [36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66], "partitioninfo": [36, 37, 38, 39, 40, 41, 43, 44, 45, 47, 48, 49, 50, 51, 53, 55, 56, 57, 58, 59, 61, 62, 63, 66], "process_func": [36, 37, 38, 49, 50, 56, 77, 78, 79, 93, 94, 97], "recordbatchread": [36, 38, 49, 77, 79, 93], "iter": [36, 38, 49, 77, 79, 93, 102], "background_io_thread": [36, 38, 49, 77, 79, 93], "streaming_batch_s": [36, 38, 49, 77, 79, 93], "secs_checkpoint_interv": [36, 38, 49, 77, 79, 93], "parquet_row_group_s": [36, 37, 38, 43, 44, 45, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102], "parquet_dictionary_encod": [36, 37, 38, 43, 44, 45, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102], "parquet_compress": [36, 37, 38, 43, 44, 45, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102], "zstd": [36, 37, 38, 43, 44, 45, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102], "parquet_compression_level": [36, 37, 38, 43, 44, 45, 49, 50, 62, 77, 78, 79, 86, 93, 94, 101, 102], "use_duckdb_read": [36, 37, 38, 49, 50, 77, 78, 79, 93, 94], "output_nam": [36, 37, 38, 43, 44, 45, 48, 49, 50, 53, 56, 62, 63, 77, 78, 79, 85, 86, 91, 93, 94, 95, 97, 98, 101, 102], "output_path": [36, 37, 38, 39, 43, 44, 45, 48, 49, 50, 53, 56, 60, 62, 63, 77, 78, 79, 83, 85, 86, 91, 93, 94, 95, 97, 98, 101, 102], "cpu_limit": [36, 37, 38, 41, 43, 44, 45, 47, 48, 49, 50, 53, 56, 58, 62, 63, 66, 77, 78, 79, 85, 86, 88, 91, 93, 94, 95, 97, 98, 99, 101, 102, 104], "gpu_limit": [36, 37, 38, 48, 49, 50, 56, 62, 63, 77, 78, 79, 91, 93, 94, 97], "float": [36, 37, 38, 48, 49, 50, 56, 60, 62, 63, 67, 77, 78, 79, 91, 93, 94, 97, 102], "memory_limit": [36, 37, 38, 41, 43, 44, 45, 47, 48, 49, 50, 53, 56, 58, 62, 63, 66, 77, 78, 79, 85, 86, 88, 91, 93, 94, 95, 97, 98, 99, 101, 102, 104], "link_manifest": 39, "copi": [39, 83], "link_or_copi": [39, 83], "manifest": [39, 83], "manifest_onli": [39, 83], "is_final_nod": [39, 83], "dimens": [41, 43, 44, 45, 47, 52, 53, 58, 66, 80, 85, 86, 88, 95, 98, 99, 101, 103, 104, 105], "partition_by_row": [41, 82, 85], "random_shuffl": [41, 43, 44, 45, 82, 85, 86, 101], "root_task": 42, "roottask": 42, "logical_plan": [42, 110], "hash_column": [43, 44, 45, 86, 101], "data_partition_column": [43, 44, 45, 47, 82, 86, 88, 101], "shuffle_onli": [43, 44, 45, 86, 101], "drop_partition_column": [43, 44, 45, 86, 101], "use_parquet_writ": [43, 44, 45, 86, 101], "hive_partit": [43, 44, 45, 47, 86, 88, 101], "hex": 46, "byte": 46, "bytes_l": 46, "field": 46, "is_saf": 46, "safeuuid": 46, "unknown": 46, "uniqu": [46, 64, 65, 67, 88, 92, 95, 98, 99, 104, 105], "identifi": [46, 64, 65, 92], "uuid": [46, 67], "either": 46, "string": [46, 83, 91, 102], "32": 46, "hexadecim": 46, "digit": 46, "16": [46, 98], "littl": 46, "endian": 46, "order": [46, 90], "tupl": [46, 77, 78, 79, 82, 83, 85, 86, 88, 91, 93, 94, 95, 97, 98, 99, 100, 101, 102, 103, 104], "six": 46, "integ": 46, "bit": 46, "time_low": 46, "time_mid": 46, "time_hi_vers": 46, "8": [46, 108], "clock_seq_hi_vari": 46, "clock_seq_low": 46, "48": 46, "singl": [46, 94, 110], "128": 46, "when": [46, 96, 102, 108], "curli": 46, "brace": 46, "hyphen": 46, "urn": 46, "prefix": [46, 91], "yield": 46, "12345678": 46, "1234": 46, "5678": 46, "567812345678": 46, "12345678123456781234567812345678": 46, "x12x34x56x78": 46, "4": 46, "x78x56x34x12x34x12x78x56": 46, "x12x34x56x78x12x34x56x78": 46, "0x12345678": 46, "0x1234": 46, "0x5678": 46, "0x12": 46, "0x34": 46, "0x567812345678": 46, "0x12345678123456781234567812345678": 46, "exactli": 46, "its": 46, "variant": 46, "accord": 46, "rfc": 46, "4122": 46, "overrid": [46, 78, 79, 97], "enum": 46, "expos": 46, "instanc": [46, 78, 79, 84, 97, 108], "indic": 46, "ha": [46, 77, 78, 79, 93, 94, 97, 110], "been": [46, 78, 79], "wai": 46, "safe": 46, "multiprocess": 46, "applic": 46, "uuid_generate_time_saf": 46, "partitionproducertask": 51, "index": [52, 102], "toplevel": 52, "inform": 52, "about": [52, 108], "cnt": 54, "sum": 54, "min": [54, 102], "max": [54, 77, 78, 79, 91, 93, 94, 102], "avg": 54, "p50": 54, "p75": 54, "p95": 54, "p99": 54, "statist": 54, "generated_column": [55, 74, 96], "job_id": [60, 67, 91, 110], "job_tim": [60, 67, 110], "datetim": 60, "num_executor": [60, 67], "random_se": 60, "env_overrid": 60, "bind_numa_nod": [60, 67], "enforce_memory_limit": 60, "max_usable_cpu_count": 60, "1024": 60, "max_usable_gpu_count": 60, "max_usable_memory_s": 60, "17592186044416": 60, "secs_wq_poll_interv": 60, "secs_executor_probe_interv": 60, "30": 60, "max_num_missed_prob": 60, "6": 60, "fault_inject_prob": 60, "enable_profil": 60, "enable_diagnostic_metr": 60, "remove_empty_parquet": 60, "skip_task_with_empty_input": 60, "shared_log_root": 60, "console_log_level": 60, "info": 60, "file_log_level": 60, "debug": 60, "disable_log_rot": 60, "state": [60, 110], "sql_queri": [62, 76, 102], "udfcontext": 62, "per_thread_output": [62, 102], "materialize_output": [62, 102], "materialize_in_memori": [62, 102], "batched_process": [62, 102], "enable_temp_directori": [62, 102], "cpu_overcommit_ratio": [62, 102], "memory_overcommit_ratio": [62, 102], "9": [62, 102], "base": [63, 70, 82, 86, 91, 95, 104, 107], "basic": 63, "unit": [63, 82, 88, 95, 98, 99, 104, 105], "produc": 63, "depend": [63, 83, 84, 109], "other": [63, 77, 78, 79, 83, 84, 86, 91, 93, 94, 97, 101, 102], "resourc": [63, 77, 78, 79, 91, 93, 94, 102], "requir": [63, 77, 78, 79, 88, 91, 93, 94, 95, 98, 99, 102, 104, 105, 108], "cpu": [63, 77, 78, 79, 91, 93, 94, 102], "gpu": [63, 77, 78, 79, 91, 93, 94], "memori": [63, 67, 77, 78, 79, 91, 93, 94, 102, 109], "idempot": 63, "retri": [63, 65, 102], "fail": 63, "lifetim": 63, "instanti": 63, "pickl": [63, 110], "sent": 63, "worker": [63, 67, 110], "call": [63, 110], "prepar": 63, "final": [63, 102], "cleanup": 63, "releas": 63, "back": 63, "taskid": 65, "epoch": [65, 67], "besid": 65, "also": [65, 102], "count": [65, 102], "partition_func": 66, "job_nam": 67, "ray_address": 67, "_remove_output_root": 67, "session": [67, 108], "pleas": [67, 108], "variabl": 67, "sp_jobid": 67, "random": [67, 82, 85, 86, 101, 102], "sp_job_tim": 67, "second": [67, 77, 79, 93], "sinc": [67, 102], "sp_job_nam": 67, "displai": 67, "name": [67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 82, 86, 91, 96, 101, 102], "filenam": [67, 91, 96], "sp_data_root": 67, "folder": [67, 83, 91], "sp_num_executor": 67, "mean": 67, "sp_ray_address": 67, "address": 67, "connect": 67, "exist": [67, 88, 95, 98, 99, 104, 105], "otherwis": [67, 83], "sp_bind_numa_nod": 67, "bind": [67, 77, 78, 79, 93, 94, 97], "numa": 67, "memory_alloc": 67, "sp_memory_alloc": 67, "alloc": 67, "choic": 67, "system": [67, 109], "jemalloc": 67, "mimalloc": 67, "remov": 67, "directori": [67, 91, 110], "finish": 67, "compat": 67, "save": 67, "sp_spawn": 67, "pattern": [68, 69, 70, 71, 72, 73, 74, 75, 76], "e": [68, 69, 70, 71, 72, 73, 74, 75, 76, 78, 79, 91, 97, 102, 109], "g": [68, 69, 70, 71, 72, 73, 74, 75, 76, 78, 79, 91, 97, 102, 109], "100": [68, 69, 70, 71, 72, 73, 74, 75, 76], "dataseta": [68, 69, 70, 71, 72, 73, 74, 75, 76], "root_dir": [68, 69, 70, 71, 72, 73, 74, 75, 76], "rel": [68, 69, 70, 71, 72, 73, 74, 75, 76], "resolv": [68, 69, 70, 71, 72, 73, 74, 75, 76], "http": [68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 88, 93, 94, 95, 96, 98, 99, 101, 102, 104, 105, 108], "org": [68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 88, 93, 94, 95, 96, 98, 99, 101, 102, 104, 105], "doc": [68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 86, 93, 94, 96, 101, 102], "multiple_fil": [68, 69, 70, 71, 72, 73, 74, 75, 76, 96], "combining_schema": [68, 69, 70, 71, 72, 73, 74, 75, 76, 96], "union": [68, 69, 70, 71, 72, 73, 74, 75, 76, 96, 103], "max_line_s": 69, "parallel": [69, 102], "header": 69, "format": [72, 91, 109], "newline_delimit": 72, "max_object_s": 72, "1073741824": 72, "query_build": 76, "duckdbpyconnect": 76, "abstractfilesystem": 76, "arrowstreamnod": [77, 93], "find": [77, 78, 79, 83, 84, 86, 93, 94, 97, 101, 102], "comment": [77, 78, 79, 83, 84, 86, 93, 94, 97, 101, 102], "signatur": [77, 78, 79, 93, 94, 97], "extra": [77, 78, 79, 93, 94, 97], "functool": [77, 78, 79, 93, 94, 97], "partial": [77, 78, 79, 88, 93, 94, 95, 97, 98, 99, 104, 105], "test_partial_process_func": [77, 78, 79, 93, 94, 97], "test": [77, 78, 79, 82, 88, 93, 94, 95, 97, 98, 99, 104, 105], "test_execut": [77, 78, 79, 93, 94, 97], "usag": [77, 78, 79, 93, 94, 97], "background": [77, 79, 93], "io": [77, 79, 83, 91, 93], "thread": [77, 79, 93, 102], "length": [77, 79, 93], "less": [77, 79, 86, 93, 101], "equal": [77, 79, 93], "checkpoint": [77, 79, 93, 110], "stream": [77, 79, 93], "everi": [77, 79, 83, 93], "store": [77, 78, 79, 82, 86, 93, 94, 101, 102, 109, 110], "group": [77, 78, 79, 80, 82, 85, 86, 93, 94, 101, 102], "larg": [77, 78, 79, 86, 93, 94, 101, 102], "opportun": [77, 78, 79, 86, 93, 94, 101, 102], "compress": [77, 78, 79, 86, 93, 94, 101, 102], "small": [77, 78, 79, 86, 93, 94, 101, 102], "could": [77, 78, 79, 86, 93, 94, 101, 102], "filter": [77, 78, 79, 86, 93, 94, 101, 102], "faster": [77, 78, 79, 86, 93, 94, 101, 102], "achiev": [77, 78, 79, 86, 93, 94, 101, 102], "high": [77, 78, 79, 86, 93, 94, 101, 102, 109], "concurr": [77, 78, 79, 86, 93, 94, 101, 102], "tip": [77, 78, 79, 86, 93, 94, 101, 102], "html": [77, 78, 79, 86, 93, 94, 101, 102], "row_group_s": [77, 78, 79, 86, 93, 94, 101, 102], "encod": [77, 78, 79, 86, 93, 94, 101, 102], "some": [77, 78, 79, 86, 93, 94, 101, 102], "use_dictionari": [77, 78, 79, 86, 93, 94, 101], "apach": [77, 78, 79, 86, 93, 94, 101], "parquetwrit": [77, 78, 79, 86, 93, 94, 101], "modul": [77, 78, 79, 93, 94], "guid": [77, 78, 79, 91, 93, 94], "doe": [77, 78, 79, 84, 91, 93, 94], "NOT": [77, 78, 79, 91, 93, 94], "enforc": [77, 78, 79, 91, 93, 94], "access": [78, 79, 83, 91, 97], "local_rank": [78, 79, 97], "random_seed_long": [78, 79, 97], "numpy_random_gen": [78, 79, 97], "subclass": [78, 79, 97], "arrowcomputetask": 78, "spawn": [78, 79, 97], "arrowstreamtask": 79, "consolid": 80, "larger": 80, "ones": 80, "effect": [80, 102], "reduc": 80, "without": [80, 109], "across": [80, 83, 91], "network": 80, "right": 81, "now": [81, 108], "mainli": 81, "keep": 81, "hashpartitionnod": [82, 101], "data_partition_column_nam": 82, "test_load_partitioned_dataset": 82, "test_partit": [82, 88, 95, 98, 99, 104, 105], "link": 83, "hard": 83, "symbol": 83, "absolut": [83, 91], "although": [83, 91], "vari": [83, 91], "filesystem": [83, 91, 102], "sink": 83, "mount": 83, "symlink": 83, "deprec": 83, "serv": 84, "dummi": 84, "nest": [85, 86, 88, 95, 98, 99, 101, 104, 105], "engine_typ": [86, 101], "proport": [86, 101], "underli": [86, 101, 102], "ignor": [86, 101], "exclud": [86, 101], "append": [86, 101], "writer": [86, 101], "intermedi": [86, 101, 109, 110], "slower": [86, 101], "hive": [86, 101], "whose": [87, 96], "subpartit": [88, 95, 98, 99, 104, 105], "test_nested_partit": [88, 95, 98, 99, 104, 105], "why": [88, 95, 98, 99, 104, 105], "advanc": [88, 95, 98, 99, 104, 105], "techniqu": [88, 95, 98, 99, 104, 105], "massiv": [88, 95, 98, 99, 104, 105], "dl": [88, 95, 98, 99, 104, 105], "acm": [88, 95, 98, 99, 104, 105], "doi": [88, 95, 98, 99, 104, 105], "1145": [88, 95, 98, 99, 104, 105], "2213836": [88, 95, 98, 99, 104, 105], "2213839": [88, 95, 98, 99, 104, 105], "root_nod": 89, "arg": 90, "kwd": 90, "visit": 90, "logcial": 90, "depth": 90, "hashpartitiontask": 91, "sqlenginetask": 91, "pythonscripttask": [91, 97], "alphanumer": 91, "charact": 91, "underscor": 91, "word": 91, "match": 91, "regular": 91, "za": 91, "z0": 91, "9_": 91, "job_root_path": 91, "task_runtime_id": 91, "seqnum": 91, "where": [91, 102], "task_id": 91, "sched_epoch": 91, "task_retry_count": 91, "calcul": 91, "ratio": 91, "machin": 91, "arrowcomputenod": [94, 96], "file_row_numb": 96, "projectnod": 96, "auto": 96, "extract": 96, "url": [96, 108], "parseurl": 96, "input_t": 96, "assert": 96, "column_nam": 96, "as_pi": 96, "maxsplit": 96, "from_arrai": 96, "suppos": 96, "sever": 96, "data_partit": 96, "Then": 96, "feed": 96, "urls_with_host": 96, "split_point": 98, "137438953472": 98, "yet": 98, "repeat": 99, "virtual": 100, "assembl": 100, "noth": 100, "userdefinedfunct": 102, "relax_memory_if_oom": 102, "extension_path": 102, "udf_module_path": 102, "against": 102, "syntax": 102, "query_syntax": 102, "But": 102, "last": [102, 110], "persist": 102, "total": 102, "product": 102, "category_id": 102, "product_item": 102, "batch_index": 102, "query_index": 102, "rand_se": 102, "seed": 102, "__data_partition__": 102, "element": 102, "decor": 102, "create_funct": 102, "create_duckdb_extens": 102, "empti": 102, "downgrad": 102, "50": 102, "gb": 102, "highli": 102, "per": 102, "significantli": 102, "improv": 102, "materi": 102, "befor": 102, "doubl": 102, "oom": 102, "fit": 102, "len": 102, "temp": [102, 110], "low": [102, 109], "too": 102, "fallback": 102, "plain": 102, "eg": 102, "column_1": 102, "dep": 103, "test_user_defined_partit": 104, "partitioned_dataset": 105, "pip": 108, "step": 108, "learn": 108, "refer": [108, 109], "core": 108, "dashboard": 108, "print": 108, "view": 108, "127": 108, "8008": 108, "lightweight": 109, "framework": 109, "3f": 109, "deliv": 109, "nativ": 109, "scalabl": 109, "leverag": 109, "pb": 109, "scale": 109, "handl": 109, "bottleneck": 109, "simplic": 109, "No": 109, "long": 109, "servic": 109, "complex": 109, "easi": 109, "deploi": 109, "maintain": 109, "get": 109, "start": 109, "intern": 109, "structur": 110, "2024": 110, "12": 110, "11": 110, "00": 110, "28": 110, "2cc39990": 110, "296f": 110, "48a3": 110, "8063": 110, "78cf6dca460b": 110, "config": 110, "log": 110, "png": 110, "queue": 110, "messag": 110, "stage": 110, "datasourcetask": 110, "000001": 110, "evenlydistributedpartitionproducertask": 110, "000002": 110, "completed_task": 110, "started_task": 110, "temporari": 110, "recov": 110, "resum": 110, "few": 110, "arrowbatchtask": 110}, "objects": {"smallpond.dataframe.DataFrame": [[6, 0, 1, "", "compute"], [7, 0, 1, "", "count"], [8, 0, 1, "", "filter"], [9, 0, 1, "", "flat_map"], [10, 0, 1, "", "is_computed"], [11, 0, 1, "", "limit"], [12, 0, 1, "", "map"], [13, 0, 1, "", "map_batches"], [14, 0, 1, "", "partial_sort"], [15, 0, 1, "", "random_shuffle"], [16, 0, 1, "", "recompute"], [17, 0, 1, "", "repartition"], [18, 0, 1, "", "take"], [19, 0, 1, "", "take_all"], [20, 0, 1, "", "to_arrow"], [21, 0, 1, "", "to_pandas"], [22, 0, 1, "", "write_parquet"], [23, 0, 1, "", "write_parquet_lazy"]], "smallpond.dataframe.Session": [[24, 0, 1, "", "from_arrow"], [25, 0, 1, "", "from_items"], [26, 0, 1, "", "from_pandas"], [27, 0, 1, "", "partial_sql"], [28, 0, 1, "", "read_csv"], [29, 0, 1, "", "read_json"], [30, 0, 1, "", "read_parquet"], [31, 0, 1, "", "wait"]], "smallpond.execution.driver": [[32, 1, 1, "", "Driver"]], "smallpond.execution.driver.Driver": [[32, 0, 1, "", "__init__"]], "smallpond.execution.executor": [[33, 1, 1, "", "Executor"]], "smallpond.execution.executor.Executor": [[33, 0, 1, "", "__init__"]], "smallpond.execution.manager": [[34, 1, 1, "", "JobManager"]], "smallpond.execution.manager.JobManager": [[34, 0, 1, "", "__init__"]], "smallpond.execution.scheduler": [[35, 1, 1, "", "Scheduler"]], "smallpond.execution.scheduler.Scheduler": [[35, 0, 1, "", "__init__"]], "smallpond.execution.task": [[36, 1, 1, "", "ArrowBatchTask"], [37, 1, 1, "", "ArrowComputeTask"], [38, 1, 1, "", "ArrowStreamTask"], [39, 1, 1, "", "DataSinkTask"], [40, 1, 1, "", "DataSourceTask"], [41, 1, 1, "", "EvenlyDistributedPartitionProducerTask"], [42, 1, 1, "", "ExecutionPlan"], [43, 1, 1, "", "HashPartitionArrowTask"], [44, 1, 1, "", "HashPartitionDuckDbTask"], [45, 1, 1, "", "HashPartitionTask"], [46, 1, 1, "", "JobId"], [47, 1, 1, "", "LoadPartitionedDataSetProducerTask"], [48, 1, 1, "", "MergeDataSetsTask"], [49, 1, 1, "", "PandasBatchTask"], [50, 1, 1, "", "PandasComputeTask"], [51, 1, 1, "", "PartitionConsumerTask"], [52, 1, 1, "", "PartitionInfo"], [53, 1, 1, "", "PartitionProducerTask"], [54, 1, 1, "", "PerfStats"], [55, 1, 1, "", "ProjectionTask"], [56, 1, 1, "", "PythonScriptTask"], [57, 1, 1, "", "RangePartitionTask"], [58, 1, 1, "", "RepeatPartitionProducerTask"], [59, 1, 1, "", "RootTask"], [60, 1, 1, "", "RuntimeContext"], [61, 1, 1, "", "SplitDataSetTask"], [62, 1, 1, "", "SqlEngineTask"], [63, 1, 1, "", "Task"], [64, 1, 1, "", "TaskId"], [65, 1, 1, "", "TaskRuntimeId"], [66, 1, 1, "", "UserDefinedPartitionProducerTask"]], "smallpond.execution.task.ArrowBatchTask": [[36, 0, 1, "", "__init__"]], "smallpond.execution.task.ArrowComputeTask": [[37, 0, 1, "", "__init__"]], "smallpond.execution.task.ArrowStreamTask": [[38, 0, 1, "", "__init__"]], "smallpond.execution.task.DataSinkTask": [[39, 0, 1, "", "__init__"]], "smallpond.execution.task.DataSourceTask": [[40, 0, 1, "", "__init__"]], "smallpond.execution.task.EvenlyDistributedPartitionProducerTask": [[41, 0, 1, "", "__init__"]], "smallpond.execution.task.ExecutionPlan": [[42, 0, 1, "", "__init__"]], "smallpond.execution.task.HashPartitionArrowTask": [[43, 0, 1, "", "__init__"]], "smallpond.execution.task.HashPartitionDuckDbTask": [[44, 0, 1, "", "__init__"]], "smallpond.execution.task.HashPartitionTask": [[45, 0, 1, "", "__init__"]], "smallpond.execution.task.JobId": [[46, 0, 1, "", "__init__"]], "smallpond.execution.task.LoadPartitionedDataSetProducerTask": [[47, 0, 1, "", "__init__"]], "smallpond.execution.task.MergeDataSetsTask": [[48, 0, 1, "", "__init__"]], "smallpond.execution.task.PandasBatchTask": [[49, 0, 1, "", "__init__"]], "smallpond.execution.task.PandasComputeTask": [[50, 0, 1, "", "__init__"]], "smallpond.execution.task.PartitionConsumerTask": [[51, 0, 1, "", "__init__"]], "smallpond.execution.task.PartitionInfo": [[52, 0, 1, "", "__init__"]], "smallpond.execution.task.PartitionProducerTask": [[53, 0, 1, "", "__init__"]], "smallpond.execution.task.PerfStats": [[54, 0, 1, "", "__init__"]], "smallpond.execution.task.ProjectionTask": [[55, 0, 1, "", "__init__"]], "smallpond.execution.task.PythonScriptTask": [[56, 0, 1, "", "__init__"]], "smallpond.execution.task.RangePartitionTask": [[57, 0, 1, "", "__init__"]], "smallpond.execution.task.RepeatPartitionProducerTask": [[58, 0, 1, "", "__init__"]], "smallpond.execution.task.RootTask": [[59, 0, 1, "", "__init__"]], "smallpond.execution.task.RuntimeContext": [[60, 0, 1, "", "__init__"]], "smallpond.execution.task.SplitDataSetTask": [[61, 0, 1, "", "__init__"]], "smallpond.execution.task.SqlEngineTask": [[62, 0, 1, "", "__init__"]], "smallpond.execution.task.Task": [[63, 0, 1, "", "__init__"]], "smallpond.execution.task.TaskId": [[64, 0, 1, "", "__init__"]], "smallpond.execution.task.TaskRuntimeId": [[65, 0, 1, "", "__init__"]], "smallpond.execution.task.UserDefinedPartitionProducerTask": [[66, 0, 1, "", "__init__"]], "smallpond": [[67, 2, 1, "", "init"]], "smallpond.logical.dataset": [[68, 1, 1, "", "ArrowTableDataSet"], [69, 1, 1, "", "CsvDataSet"], [70, 1, 1, "", "DataSet"], [71, 1, 1, "", "FileSet"], [72, 1, 1, "", "JsonDataSet"], [73, 1, 1, "", "PandasDataSet"], [74, 1, 1, "", "ParquetDataSet"], [75, 1, 1, "", "PartitionedDataSet"], [76, 1, 1, "", "SqlQueryDataSet"]], "smallpond.logical.dataset.ArrowTableDataSet": [[68, 0, 1, "", "__init__"]], "smallpond.logical.dataset.CsvDataSet": [[69, 0, 1, "", "__init__"]], "smallpond.logical.dataset.DataSet": [[70, 0, 1, "", "__init__"]], "smallpond.logical.dataset.FileSet": [[71, 0, 1, "", "__init__"]], "smallpond.logical.dataset.JsonDataSet": [[72, 0, 1, "", "__init__"]], "smallpond.logical.dataset.PandasDataSet": [[73, 0, 1, "", "__init__"]], "smallpond.logical.dataset.ParquetDataSet": [[74, 0, 1, "", "__init__"]], "smallpond.logical.dataset.PartitionedDataSet": [[75, 0, 1, "", "__init__"]], "smallpond.logical.dataset.SqlQueryDataSet": [[76, 0, 1, "", "__init__"]], "smallpond.logical.node": [[77, 1, 1, "", "ArrowBatchNode"], [78, 1, 1, "", "ArrowComputeNode"], [79, 1, 1, "", "ArrowStreamNode"], [80, 1, 1, "", "ConsolidateNode"], [81, 1, 1, "", "Context"], [82, 2, 1, "", "DataSetPartitionNode"], [83, 1, 1, "", "DataSinkNode"], [84, 1, 1, "", "DataSourceNode"], [85, 1, 1, "", "EvenlyDistributedPartitionNode"], [86, 1, 1, "", "HashPartitionNode"], [87, 1, 1, "", "LimitNode"], [88, 1, 1, "", "LoadPartitionedDataSetNode"], [89, 1, 1, "", "LogicalPlan"], [90, 1, 1, "", "LogicalPlanVisitor"], [91, 1, 1, "", "Node"], [92, 1, 1, "", "NodeId"], [93, 1, 1, "", "PandasBatchNode"], [94, 1, 1, "", "PandasComputeNode"], [95, 1, 1, "", "PartitionNode"], [96, 1, 1, "", "ProjectionNode"], [97, 1, 1, "", "PythonScriptNode"], [98, 1, 1, "", "RangePartitionNode"], [99, 1, 1, "", "RepeatPartitionNode"], [100, 1, 1, "", "RootNode"], [101, 1, 1, "", "ShuffleNode"], [102, 1, 1, "", "SqlEngineNode"], [103, 1, 1, "", "UnionNode"], [104, 1, 1, "", "UserDefinedPartitionNode"], [105, 1, 1, "", "UserPartitionedDataSourceNode"]], "smallpond.logical.node.ArrowBatchNode": [[77, 0, 1, "", "__init__"]], "smallpond.logical.node.ArrowComputeNode": [[78, 0, 1, "", "__init__"]], "smallpond.logical.node.ArrowStreamNode": [[79, 0, 1, "", "__init__"]], "smallpond.logical.node.ConsolidateNode": [[80, 0, 1, "", "__init__"]], "smallpond.logical.node.Context": [[81, 0, 1, "", "__init__"]], "smallpond.logical.node.DataSinkNode": [[83, 0, 1, "", "__init__"]], "smallpond.logical.node.DataSourceNode": [[84, 0, 1, "", "__init__"]], "smallpond.logical.node.EvenlyDistributedPartitionNode": [[85, 0, 1, "", "__init__"]], "smallpond.logical.node.HashPartitionNode": [[86, 0, 1, "", "__init__"]], "smallpond.logical.node.LimitNode": [[87, 0, 1, "", "__init__"]], "smallpond.logical.node.LoadPartitionedDataSetNode": [[88, 0, 1, "", "__init__"]], "smallpond.logical.node.LogicalPlan": [[89, 0, 1, "", "__init__"]], "smallpond.logical.node.LogicalPlanVisitor": [[90, 0, 1, "", "__init__"]], "smallpond.logical.node.Node": [[91, 0, 1, "", "__init__"]], "smallpond.logical.node.NodeId": [[92, 0, 1, "", "__init__"]], "smallpond.logical.node.PandasBatchNode": [[93, 0, 1, "", "__init__"]], "smallpond.logical.node.PandasComputeNode": [[94, 0, 1, "", "__init__"]], "smallpond.logical.node.PartitionNode": [[95, 0, 1, "", "__init__"]], "smallpond.logical.node.ProjectionNode": [[96, 0, 1, "", "__init__"]], "smallpond.logical.node.PythonScriptNode": [[97, 0, 1, "", "__init__"]], "smallpond.logical.node.RangePartitionNode": [[98, 0, 1, "", "__init__"]], "smallpond.logical.node.RepeatPartitionNode": [[99, 0, 1, "", "__init__"]], "smallpond.logical.node.RootNode": [[100, 0, 1, "", "__init__"]], "smallpond.logical.node.ShuffleNode": [[101, 0, 1, "", "__init__"]], "smallpond.logical.node.SqlEngineNode": [[102, 0, 1, "", "__init__"]], "smallpond.logical.node.UnionNode": [[103, 0, 1, "", "__init__"]], "smallpond.logical.node.UserDefinedPartitionNode": [[104, 0, 1, "", "__init__"]], "smallpond.logical.node.UserPartitionedDataSourceNode": [[105, 0, 1, "", "__init__"]], "smallpond.platform": [[106, 1, 1, "", "MPI"], [107, 1, 1, "", "Platform"]], "smallpond.platform.MPI": [[106, 0, 1, "", "__init__"]], "smallpond.platform.Platform": [[107, 0, 1, "", "__init__"]]}, "objtypes": {"0": "py:method", "1": "py:class", "2": "py:function"}, "objnames": {"0": ["py", "method", "Python method"], "1": ["py", "class", "Python class"], "2": ["py", "function", "Python function"]}, "titleterms": {"api": 0, "refer": 0, "high": 0, "level": 0, "low": 0, "datafram": [1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "initi": [1, 108], "load": [1, 108], "data": [1, 108, 110], "partit": [1, 108], "transform": [1, 108], "consum": 1, "execut": [1, 3, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66], "dataset": [2, 68, 69, 70, 71, 72, 73, 74, 75, 76], "submit": 3, "job": [3, 67], "schedul": [3, 35], "executor": [3, 33], "custom": 3, "platform": [3, 106, 107], "node": [4, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "context": [4, 81], "logicalplan": [4, 89], "task": [5, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66], "runtimecontext": [5, 60], "executionplan": [5, 42], "smallpond": [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 109], "comput": 6, "count": 7, "filter": 8, "paramet": [8, 9, 12, 13, 14, 17, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 91, 93, 94, 95, 96, 97, 98, 99, 101, 102, 103, 104, 105], "exampl": [8, 9, 12, 14, 16, 17, 22, 23, 27, 31, 82, 88, 95, 96, 98, 99, 104, 105], "flat_map": 9, "is_comput": 10, "limit": 11, "map": 12, "map_batch": 13, "partial_sort": 14, "random_shuffl": 15, "recomput": 16, "repartit": 17, "take": 18, "take_al": 19, "to_arrow": 20, "to_panda": 21, "write_parquet": 22, "write_parquet_lazi": 23, "session": [24, 25, 26, 27, 28, 29, 30, 31], "from_arrow": 24, "from_item": 25, "from_panda": 26, "partial_sql": 27, "read_csv": 28, "read_json": 29, "read_parquet": 30, "wait": 31, "driver": 32, "manag": 34, "jobmanag": 34, "arrowbatchtask": 36, "arrowcomputetask": 37, "arrowstreamtask": 38, "datasinktask": 39, "datasourcetask": 40, "evenlydistributedpartitionproducertask": 41, "hashpartitionarrowtask": 43, "hashpartitionduckdbtask": 44, "hashpartitiontask": 45, "jobid": 46, "loadpartitioneddatasetproducertask": 47, "mergedatasetstask": 48, "pandasbatchtask": 49, "pandascomputetask": 50, "partitionconsumertask": 51, "partitioninfo": 52, "partitionproducertask": 53, "perfstat": 54, "projectiontask": 55, "pythonscripttask": 56, "rangepartitiontask": 57, "repeatpartitionproducertask": 58, "roottask": 59, "splitdatasettask": 61, "sqlenginetask": 62, "taskid": 64, "taskruntimeid": 65, "userdefinedpartitionproducertask": 66, "init": 67, "spawn": 67, "new": 67, "logic": [68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105], "arrowtabledataset": 68, "csvdataset": 69, "fileset": 71, "jsondataset": 72, "pandasdataset": 73, "parquetdataset": 74, "partitioneddataset": 75, "sqlquerydataset": 76, "arrowbatchnod": 77, "arrowcomputenod": 78, "arrowstreamnod": 79, "consolidatenod": 80, "datasetpartitionnod": 82, "datasinknod": 83, "datasourcenod": 84, "evenlydistributedpartitionnod": 85, "hashpartitionnod": 86, "limitnod": 87, "loadpartitioneddatasetnod": 88, "logicalplanvisitor": 90, "nodeid": 92, "pandasbatchnod": 93, "pandascomputenod": 94, "partitionnod": 95, "projectionnod": 96, "pythonscriptnod": 97, "rangepartitionnod": 98, "repeatpartitionnod": 99, "rootnod": 100, "shufflenod": 101, "sqlenginenod": 102, "unionnod": 103, "userdefinedpartitionnod": 104, "userpartitioneddatasourcenod": 105, "mpi": 106, "get": 108, "start": 108, "instal": 108, "save": 108, "monitor": 108, "why": 109, "intern": 110, "root": 110, "failur": 110, "recoveri": 110}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 58}, "alltitles": {"API Reference": [[0, "api-reference"]], "High-level API": [[0, "high-level-api"]], "Low-level API": [[0, "low-level-api"]], "DataFrame": [[1, "dataframe"]], "Initialization": [[1, "initialization"], [108, "initialization"]], "Loading Data": [[1, "loading-data"], [108, "loading-data"]], "Partitioning Data": [[1, "partitioning-data"], [108, "partitioning-data"]], "Transformations": [[1, "transformations"]], "Consuming Data": [[1, "consuming-data"]], "Execution": [[1, "execution"], [3, "execution"]], "Dataset": [[2, "dataset"]], "DataSets": [[2, "datasets"]], "Submit a Job": [[3, "submit-a-job"]], "Scheduler and Executor": [[3, "scheduler-and-executor"]], "Customize Platform": [[3, "customize-platform"]], "Nodes": [[4, "nodes"], [4, "id2"]], "Context": [[4, "context"]], "LogicalPlan": [[4, "logicalplan"]], "Tasks": [[5, "tasks"], [5, "id2"]], "RuntimeContext": [[5, "runtimecontext"]], "ExecutionPlan": [[5, "executionplan"]], "smallpond.dataframe.DataFrame.compute": [[6, "smallpond-dataframe-dataframe-compute"]], "smallpond.dataframe.DataFrame.count": [[7, "smallpond-dataframe-dataframe-count"]], "smallpond.dataframe.DataFrame.filter": [[8, "smallpond-dataframe-dataframe-filter"]], "Parameters": [[8, "parameters"], [9, "parameters"], [12, "parameters"], [13, "parameters"], [14, "parameters"], [17, "parameters"], [67, "parameters"], [68, "parameters"], [69, "parameters"], [70, "parameters"], [71, "parameters"], [72, "parameters"], [73, "parameters"], [74, "parameters"], [75, "parameters"], [76, "parameters"], [77, "parameters"], [78, "parameters"], [79, "parameters"], [80, "parameters"], [82, "parameters"], [83, "parameters"], [84, "parameters"], [85, "parameters"], [86, "parameters"], [87, "parameters"], [88, "parameters"], [91, "parameters"], [93, "parameters"], [94, "parameters"], [95, "parameters"], [96, "parameters"], [97, "parameters"], [98, "parameters"], [99, "parameters"], [101, "parameters"], [102, "parameters"], [103, "parameters"], [104, "parameters"], [105, "parameters"]], "Examples": [[8, "examples"], [9, "examples"], [12, "examples"], [14, "examples"], [16, "examples"], [17, "examples"], [22, "examples"], [23, "examples"], [27, "examples"], [82, "examples"], [88, "examples"], [95, "examples"], [96, "examples"], [98, "examples"], [99, "examples"], [104, "examples"], [105, "examples"]], "smallpond.dataframe.DataFrame.flat_map": [[9, "smallpond-dataframe-dataframe-flat-map"]], "smallpond.dataframe.DataFrame.is_computed": [[10, "smallpond-dataframe-dataframe-is-computed"]], "smallpond.dataframe.DataFrame.limit": [[11, "smallpond-dataframe-dataframe-limit"]], "smallpond.dataframe.DataFrame.map": [[12, "smallpond-dataframe-dataframe-map"]], "smallpond.dataframe.DataFrame.map_batches": [[13, "smallpond-dataframe-dataframe-map-batches"]], "smallpond.dataframe.DataFrame.partial_sort": [[14, "smallpond-dataframe-dataframe-partial-sort"]], "smallpond.dataframe.DataFrame.random_shuffle": [[15, "smallpond-dataframe-dataframe-random-shuffle"]], "smallpond.dataframe.DataFrame.recompute": [[16, "smallpond-dataframe-dataframe-recompute"]], "smallpond.dataframe.DataFrame.repartition": [[17, "smallpond-dataframe-dataframe-repartition"]], "smallpond.dataframe.DataFrame.take": [[18, "smallpond-dataframe-dataframe-take"]], "smallpond.dataframe.DataFrame.take_all": [[19, "smallpond-dataframe-dataframe-take-all"]], "smallpond.dataframe.DataFrame.to_arrow": [[20, "smallpond-dataframe-dataframe-to-arrow"]], "smallpond.dataframe.DataFrame.to_pandas": [[21, "smallpond-dataframe-dataframe-to-pandas"]], "smallpond.dataframe.DataFrame.write_parquet": [[22, "smallpond-dataframe-dataframe-write-parquet"]], "smallpond.dataframe.DataFrame.write_parquet_lazy": [[23, "smallpond-dataframe-dataframe-write-parquet-lazy"]], "smallpond.dataframe.Session.from_arrow": [[24, "smallpond-dataframe-session-from-arrow"]], "smallpond.dataframe.Session.from_items": [[25, "smallpond-dataframe-session-from-items"]], "smallpond.dataframe.Session.from_pandas": [[26, "smallpond-dataframe-session-from-pandas"]], "smallpond.dataframe.Session.partial_sql": [[27, "smallpond-dataframe-session-partial-sql"]], "smallpond.dataframe.Session.read_csv": [[28, "smallpond-dataframe-session-read-csv"]], "smallpond.dataframe.Session.read_json": [[29, "smallpond-dataframe-session-read-json"]], "smallpond.dataframe.Session.read_parquet": [[30, "smallpond-dataframe-session-read-parquet"]], "smallpond.dataframe.Session.wait": [[31, "smallpond-dataframe-session-wait"]], "Example": [[31, "example"]], "smallpond.execution.driver.Driver": [[32, "smallpond-execution-driver-driver"]], "smallpond.execution.executor.Executor": [[33, "smallpond-execution-executor-executor"]], "smallpond.execution.manager.JobManager": [[34, "smallpond-execution-manager-jobmanager"]], "smallpond.execution.scheduler.Scheduler": [[35, "smallpond-execution-scheduler-scheduler"]], "smallpond.execution.task.ArrowBatchTask": [[36, "smallpond-execution-task-arrowbatchtask"]], "smallpond.execution.task.ArrowComputeTask": [[37, "smallpond-execution-task-arrowcomputetask"]], "smallpond.execution.task.ArrowStreamTask": [[38, "smallpond-execution-task-arrowstreamtask"]], "smallpond.execution.task.DataSinkTask": [[39, "smallpond-execution-task-datasinktask"]], "smallpond.execution.task.DataSourceTask": [[40, "smallpond-execution-task-datasourcetask"]], "smallpond.execution.task.EvenlyDistributedPartitionProducerTask": [[41, "smallpond-execution-task-evenlydistributedpartitionproducertask"]], "smallpond.execution.task.ExecutionPlan": [[42, "smallpond-execution-task-executionplan"]], "smallpond.execution.task.HashPartitionArrowTask": [[43, "smallpond-execution-task-hashpartitionarrowtask"]], "smallpond.execution.task.HashPartitionDuckDbTask": [[44, "smallpond-execution-task-hashpartitionduckdbtask"]], "smallpond.execution.task.HashPartitionTask": [[45, "smallpond-execution-task-hashpartitiontask"]], "smallpond.execution.task.JobId": [[46, "smallpond-execution-task-jobid"]], "smallpond.execution.task.LoadPartitionedDataSetProducerTask": [[47, "smallpond-execution-task-loadpartitioneddatasetproducertask"]], "smallpond.execution.task.MergeDataSetsTask": [[48, "smallpond-execution-task-mergedatasetstask"]], "smallpond.execution.task.PandasBatchTask": [[49, "smallpond-execution-task-pandasbatchtask"]], "smallpond.execution.task.PandasComputeTask": [[50, "smallpond-execution-task-pandascomputetask"]], "smallpond.execution.task.PartitionConsumerTask": [[51, "smallpond-execution-task-partitionconsumertask"]], "smallpond.execution.task.PartitionInfo": [[52, "smallpond-execution-task-partitioninfo"]], "smallpond.execution.task.PartitionProducerTask": [[53, "smallpond-execution-task-partitionproducertask"]], "smallpond.execution.task.PerfStats": [[54, "smallpond-execution-task-perfstats"]], "smallpond.execution.task.ProjectionTask": [[55, "smallpond-execution-task-projectiontask"]], "smallpond.execution.task.PythonScriptTask": [[56, "smallpond-execution-task-pythonscripttask"]], "smallpond.execution.task.RangePartitionTask": [[57, "smallpond-execution-task-rangepartitiontask"]], "smallpond.execution.task.RepeatPartitionProducerTask": [[58, "smallpond-execution-task-repeatpartitionproducertask"]], "smallpond.execution.task.RootTask": [[59, "smallpond-execution-task-roottask"]], "smallpond.execution.task.RuntimeContext": [[60, "smallpond-execution-task-runtimecontext"]], "smallpond.execution.task.SplitDataSetTask": [[61, "smallpond-execution-task-splitdatasettask"]], "smallpond.execution.task.SqlEngineTask": [[62, "smallpond-execution-task-sqlenginetask"]], "smallpond.execution.task.Task": [[63, "smallpond-execution-task-task"]], "smallpond.execution.task.TaskId": [[64, "smallpond-execution-task-taskid"]], "smallpond.execution.task.TaskRuntimeId": [[65, "smallpond-execution-task-taskruntimeid"]], "smallpond.execution.task.UserDefinedPartitionProducerTask": [[66, "smallpond-execution-task-userdefinedpartitionproducertask"]], "smallpond.init": [[67, "smallpond-init"]], "Spawning a new job": [[67, "spawning-a-new-job"]], "smallpond.logical.dataset.ArrowTableDataSet": [[68, "smallpond-logical-dataset-arrowtabledataset"]], "smallpond.logical.dataset.CsvDataSet": [[69, "smallpond-logical-dataset-csvdataset"]], "smallpond.logical.dataset.DataSet": [[70, "smallpond-logical-dataset-dataset"]], "smallpond.logical.dataset.FileSet": [[71, "smallpond-logical-dataset-fileset"]], "smallpond.logical.dataset.JsonDataSet": [[72, "smallpond-logical-dataset-jsondataset"]], "smallpond.logical.dataset.PandasDataSet": [[73, "smallpond-logical-dataset-pandasdataset"]], "smallpond.logical.dataset.ParquetDataSet": [[74, "smallpond-logical-dataset-parquetdataset"]], "smallpond.logical.dataset.PartitionedDataSet": [[75, "smallpond-logical-dataset-partitioneddataset"]], "smallpond.logical.dataset.SqlQueryDataSet": [[76, "smallpond-logical-dataset-sqlquerydataset"]], "smallpond.logical.node.ArrowBatchNode": [[77, "smallpond-logical-node-arrowbatchnode"]], "smallpond.logical.node.ArrowComputeNode": [[78, "smallpond-logical-node-arrowcomputenode"]], "smallpond.logical.node.ArrowStreamNode": [[79, "smallpond-logical-node-arrowstreamnode"]], "smallpond.logical.node.ConsolidateNode": [[80, "smallpond-logical-node-consolidatenode"]], "smallpond.logical.node.Context": [[81, "smallpond-logical-node-context"]], "smallpond.logical.node.DataSetPartitionNode": [[82, "smallpond-logical-node-datasetpartitionnode"]], "smallpond.logical.node.DataSinkNode": [[83, "smallpond-logical-node-datasinknode"]], "smallpond.logical.node.DataSourceNode": [[84, "smallpond-logical-node-datasourcenode"]], "smallpond.logical.node.EvenlyDistributedPartitionNode": [[85, "smallpond-logical-node-evenlydistributedpartitionnode"]], "smallpond.logical.node.HashPartitionNode": [[86, "smallpond-logical-node-hashpartitionnode"]], "smallpond.logical.node.LimitNode": [[87, "smallpond-logical-node-limitnode"]], "smallpond.logical.node.LoadPartitionedDataSetNode": [[88, "smallpond-logical-node-loadpartitioneddatasetnode"]], "smallpond.logical.node.LogicalPlan": [[89, "smallpond-logical-node-logicalplan"]], "smallpond.logical.node.LogicalPlanVisitor": [[90, "smallpond-logical-node-logicalplanvisitor"]], "smallpond.logical.node.Node": [[91, "smallpond-logical-node-node"]], "smallpond.logical.node.NodeId": [[92, "smallpond-logical-node-nodeid"]], "smallpond.logical.node.PandasBatchNode": [[93, "smallpond-logical-node-pandasbatchnode"]], "smallpond.logical.node.PandasComputeNode": [[94, "smallpond-logical-node-pandascomputenode"]], "smallpond.logical.node.PartitionNode": [[95, "smallpond-logical-node-partitionnode"]], "smallpond.logical.node.ProjectionNode": [[96, "smallpond-logical-node-projectionnode"]], "smallpond.logical.node.PythonScriptNode": [[97, "smallpond-logical-node-pythonscriptnode"]], "smallpond.logical.node.RangePartitionNode": [[98, "smallpond-logical-node-rangepartitionnode"]], "smallpond.logical.node.RepeatPartitionNode": [[99, "smallpond-logical-node-repeatpartitionnode"]], "smallpond.logical.node.RootNode": [[100, "smallpond-logical-node-rootnode"]], "smallpond.logical.node.ShuffleNode": [[101, "smallpond-logical-node-shufflenode"]], "smallpond.logical.node.SqlEngineNode": [[102, "smallpond-logical-node-sqlenginenode"]], "smallpond.logical.node.UnionNode": [[103, "smallpond-logical-node-unionnode"]], "smallpond.logical.node.UserDefinedPartitionNode": [[104, "smallpond-logical-node-userdefinedpartitionnode"]], "smallpond.logical.node.UserPartitionedDataSourceNode": [[105, "smallpond-logical-node-userpartitioneddatasourcenode"]], "smallpond.platform.MPI": [[106, "smallpond-platform-mpi"]], "smallpond.platform.Platform": [[107, "smallpond-platform-platform"]], "Getting Started": [[108, "getting-started"]], "Installation": [[108, "installation"]], "Transforming Data": [[108, "transforming-data"]], "Saving Data": [[108, "saving-data"]], "Monitoring": [[108, "monitoring"]], "smallpond": [[109, "smallpond"]], "Why smallpond?": [[109, "why-smallpond"]], "Internals": [[110, "internals"]], "Data Root": [[110, "data-root"]], "Failure Recovery": [[110, "failure-recovery"]]}, "indexentries": {"compute() (smallpond.dataframe.dataframe method)": [[6, "smallpond.dataframe.DataFrame.compute"]], "count() (smallpond.dataframe.dataframe method)": [[7, "smallpond.dataframe.DataFrame.count"]], "filter() (smallpond.dataframe.dataframe method)": [[8, "smallpond.dataframe.DataFrame.filter"]], "flat_map() (smallpond.dataframe.dataframe method)": [[9, "smallpond.dataframe.DataFrame.flat_map"]], "is_computed() (smallpond.dataframe.dataframe method)": [[10, "smallpond.dataframe.DataFrame.is_computed"]], "limit() (smallpond.dataframe.dataframe method)": [[11, "smallpond.dataframe.DataFrame.limit"]], "map() (smallpond.dataframe.dataframe method)": [[12, "smallpond.dataframe.DataFrame.map"]], "map_batches() (smallpond.dataframe.dataframe method)": [[13, "smallpond.dataframe.DataFrame.map_batches"]], "partial_sort() (smallpond.dataframe.dataframe method)": [[14, "smallpond.dataframe.DataFrame.partial_sort"]], "random_shuffle() (smallpond.dataframe.dataframe method)": [[15, "smallpond.dataframe.DataFrame.random_shuffle"]], "recompute() (smallpond.dataframe.dataframe method)": [[16, "smallpond.dataframe.DataFrame.recompute"]], "repartition() (smallpond.dataframe.dataframe method)": [[17, "smallpond.dataframe.DataFrame.repartition"]], "take() (smallpond.dataframe.dataframe method)": [[18, "smallpond.dataframe.DataFrame.take"]], "take_all() (smallpond.dataframe.dataframe method)": [[19, "smallpond.dataframe.DataFrame.take_all"]], "to_arrow() (smallpond.dataframe.dataframe method)": [[20, "smallpond.dataframe.DataFrame.to_arrow"]], "to_pandas() (smallpond.dataframe.dataframe method)": [[21, "smallpond.dataframe.DataFrame.to_pandas"]], "write_parquet() (smallpond.dataframe.dataframe method)": [[22, "smallpond.dataframe.DataFrame.write_parquet"]], "write_parquet_lazy() (smallpond.dataframe.dataframe method)": [[23, "smallpond.dataframe.DataFrame.write_parquet_lazy"]], "from_arrow() (smallpond.dataframe.session method)": [[24, "smallpond.dataframe.Session.from_arrow"]], "from_items() (smallpond.dataframe.session method)": [[25, "smallpond.dataframe.Session.from_items"]], "from_pandas() (smallpond.dataframe.session method)": [[26, "smallpond.dataframe.Session.from_pandas"]], "partial_sql() (smallpond.dataframe.session method)": [[27, "smallpond.dataframe.Session.partial_sql"]], "read_csv() (smallpond.dataframe.session method)": [[28, "smallpond.dataframe.Session.read_csv"]], "read_json() (smallpond.dataframe.session method)": [[29, "smallpond.dataframe.Session.read_json"]], "read_parquet() (smallpond.dataframe.session method)": [[30, "smallpond.dataframe.Session.read_parquet"]], "wait() (smallpond.dataframe.session method)": [[31, "smallpond.dataframe.Session.wait"]], "driver (class in smallpond.execution.driver)": [[32, "smallpond.execution.driver.Driver"]], "__init__() (smallpond.execution.driver.driver method)": [[32, "smallpond.execution.driver.Driver.__init__"]], "executor (class in smallpond.execution.executor)": [[33, "smallpond.execution.executor.Executor"]], "__init__() (smallpond.execution.executor.executor method)": [[33, "smallpond.execution.executor.Executor.__init__"]], "jobmanager (class in smallpond.execution.manager)": [[34, "smallpond.execution.manager.JobManager"]], "__init__() (smallpond.execution.manager.jobmanager method)": [[34, "smallpond.execution.manager.JobManager.__init__"]], "scheduler (class in smallpond.execution.scheduler)": [[35, "smallpond.execution.scheduler.Scheduler"]], "__init__() (smallpond.execution.scheduler.scheduler method)": [[35, "smallpond.execution.scheduler.Scheduler.__init__"]], "arrowbatchtask (class in smallpond.execution.task)": [[36, "smallpond.execution.task.ArrowBatchTask"]], "__init__() (smallpond.execution.task.arrowbatchtask method)": [[36, "smallpond.execution.task.ArrowBatchTask.__init__"]], "arrowcomputetask (class in smallpond.execution.task)": [[37, "smallpond.execution.task.ArrowComputeTask"]], "__init__() (smallpond.execution.task.arrowcomputetask method)": [[37, "smallpond.execution.task.ArrowComputeTask.__init__"]], "arrowstreamtask (class in smallpond.execution.task)": [[38, "smallpond.execution.task.ArrowStreamTask"]], "__init__() (smallpond.execution.task.arrowstreamtask method)": [[38, "smallpond.execution.task.ArrowStreamTask.__init__"]], "datasinktask (class in smallpond.execution.task)": [[39, "smallpond.execution.task.DataSinkTask"]], "__init__() (smallpond.execution.task.datasinktask method)": [[39, "smallpond.execution.task.DataSinkTask.__init__"]], "datasourcetask (class in smallpond.execution.task)": [[40, "smallpond.execution.task.DataSourceTask"]], "__init__() (smallpond.execution.task.datasourcetask method)": [[40, "smallpond.execution.task.DataSourceTask.__init__"]], "evenlydistributedpartitionproducertask (class in smallpond.execution.task)": [[41, "smallpond.execution.task.EvenlyDistributedPartitionProducerTask"]], "__init__() (smallpond.execution.task.evenlydistributedpartitionproducertask method)": [[41, "smallpond.execution.task.EvenlyDistributedPartitionProducerTask.__init__"]], "executionplan (class in smallpond.execution.task)": [[42, "smallpond.execution.task.ExecutionPlan"]], "__init__() (smallpond.execution.task.executionplan method)": [[42, "smallpond.execution.task.ExecutionPlan.__init__"]], "hashpartitionarrowtask (class in smallpond.execution.task)": [[43, "smallpond.execution.task.HashPartitionArrowTask"]], "__init__() (smallpond.execution.task.hashpartitionarrowtask method)": [[43, "smallpond.execution.task.HashPartitionArrowTask.__init__"]], "hashpartitionduckdbtask (class in smallpond.execution.task)": [[44, "smallpond.execution.task.HashPartitionDuckDbTask"]], "__init__() (smallpond.execution.task.hashpartitionduckdbtask method)": [[44, "smallpond.execution.task.HashPartitionDuckDbTask.__init__"]], "hashpartitiontask (class in smallpond.execution.task)": [[45, "smallpond.execution.task.HashPartitionTask"]], "__init__() (smallpond.execution.task.hashpartitiontask method)": [[45, "smallpond.execution.task.HashPartitionTask.__init__"]], "jobid (class in smallpond.execution.task)": [[46, "smallpond.execution.task.JobId"]], "__init__() (smallpond.execution.task.jobid method)": [[46, "smallpond.execution.task.JobId.__init__"]], "loadpartitioneddatasetproducertask (class in smallpond.execution.task)": [[47, "smallpond.execution.task.LoadPartitionedDataSetProducerTask"]], "__init__() (smallpond.execution.task.loadpartitioneddatasetproducertask method)": [[47, "smallpond.execution.task.LoadPartitionedDataSetProducerTask.__init__"]], "mergedatasetstask (class in smallpond.execution.task)": [[48, "smallpond.execution.task.MergeDataSetsTask"]], "__init__() (smallpond.execution.task.mergedatasetstask method)": [[48, "smallpond.execution.task.MergeDataSetsTask.__init__"]], "pandasbatchtask (class in smallpond.execution.task)": [[49, "smallpond.execution.task.PandasBatchTask"]], "__init__() (smallpond.execution.task.pandasbatchtask method)": [[49, "smallpond.execution.task.PandasBatchTask.__init__"]], "pandascomputetask (class in smallpond.execution.task)": [[50, "smallpond.execution.task.PandasComputeTask"]], "__init__() (smallpond.execution.task.pandascomputetask method)": [[50, "smallpond.execution.task.PandasComputeTask.__init__"]], "partitionconsumertask (class in smallpond.execution.task)": [[51, "smallpond.execution.task.PartitionConsumerTask"]], "__init__() (smallpond.execution.task.partitionconsumertask method)": [[51, "smallpond.execution.task.PartitionConsumerTask.__init__"]], "partitioninfo (class in smallpond.execution.task)": [[52, "smallpond.execution.task.PartitionInfo"]], "__init__() (smallpond.execution.task.partitioninfo method)": [[52, "smallpond.execution.task.PartitionInfo.__init__"]], "partitionproducertask (class in smallpond.execution.task)": [[53, "smallpond.execution.task.PartitionProducerTask"]], "__init__() (smallpond.execution.task.partitionproducertask method)": [[53, "smallpond.execution.task.PartitionProducerTask.__init__"]], "perfstats (class in smallpond.execution.task)": [[54, "smallpond.execution.task.PerfStats"]], "__init__() (smallpond.execution.task.perfstats method)": [[54, "smallpond.execution.task.PerfStats.__init__"]], "projectiontask (class in smallpond.execution.task)": [[55, "smallpond.execution.task.ProjectionTask"]], "__init__() (smallpond.execution.task.projectiontask method)": [[55, "smallpond.execution.task.ProjectionTask.__init__"]], "pythonscripttask (class in smallpond.execution.task)": [[56, "smallpond.execution.task.PythonScriptTask"]], "__init__() (smallpond.execution.task.pythonscripttask method)": [[56, "smallpond.execution.task.PythonScriptTask.__init__"]], "rangepartitiontask (class in smallpond.execution.task)": [[57, "smallpond.execution.task.RangePartitionTask"]], "__init__() (smallpond.execution.task.rangepartitiontask method)": [[57, "smallpond.execution.task.RangePartitionTask.__init__"]], "repeatpartitionproducertask (class in smallpond.execution.task)": [[58, "smallpond.execution.task.RepeatPartitionProducerTask"]], "__init__() (smallpond.execution.task.repeatpartitionproducertask method)": [[58, "smallpond.execution.task.RepeatPartitionProducerTask.__init__"]], "roottask (class in smallpond.execution.task)": [[59, "smallpond.execution.task.RootTask"]], "__init__() (smallpond.execution.task.roottask method)": [[59, "smallpond.execution.task.RootTask.__init__"]], "runtimecontext (class in smallpond.execution.task)": [[60, "smallpond.execution.task.RuntimeContext"]], "__init__() (smallpond.execution.task.runtimecontext method)": [[60, "smallpond.execution.task.RuntimeContext.__init__"]], "splitdatasettask (class in smallpond.execution.task)": [[61, "smallpond.execution.task.SplitDataSetTask"]], "__init__() (smallpond.execution.task.splitdatasettask method)": [[61, "smallpond.execution.task.SplitDataSetTask.__init__"]], "sqlenginetask (class in smallpond.execution.task)": [[62, "smallpond.execution.task.SqlEngineTask"]], "__init__() (smallpond.execution.task.sqlenginetask method)": [[62, "smallpond.execution.task.SqlEngineTask.__init__"]], "task (class in smallpond.execution.task)": [[63, "smallpond.execution.task.Task"]], "__init__() (smallpond.execution.task.task method)": [[63, "smallpond.execution.task.Task.__init__"]], "taskid (class in smallpond.execution.task)": [[64, "smallpond.execution.task.TaskId"]], "__init__() (smallpond.execution.task.taskid method)": [[64, "smallpond.execution.task.TaskId.__init__"]], "taskruntimeid (class in smallpond.execution.task)": [[65, "smallpond.execution.task.TaskRuntimeId"]], "__init__() (smallpond.execution.task.taskruntimeid method)": [[65, "smallpond.execution.task.TaskRuntimeId.__init__"]], "userdefinedpartitionproducertask (class in smallpond.execution.task)": [[66, "smallpond.execution.task.UserDefinedPartitionProducerTask"]], "__init__() (smallpond.execution.task.userdefinedpartitionproducertask method)": [[66, "smallpond.execution.task.UserDefinedPartitionProducerTask.__init__"]], "init() (in module smallpond)": [[67, "smallpond.init"]], "arrowtabledataset (class in smallpond.logical.dataset)": [[68, "smallpond.logical.dataset.ArrowTableDataSet"]], "__init__() (smallpond.logical.dataset.arrowtabledataset method)": [[68, "smallpond.logical.dataset.ArrowTableDataSet.__init__"]], "csvdataset (class in smallpond.logical.dataset)": [[69, "smallpond.logical.dataset.CsvDataSet"]], "__init__() (smallpond.logical.dataset.csvdataset method)": [[69, "smallpond.logical.dataset.CsvDataSet.__init__"]], "dataset (class in smallpond.logical.dataset)": [[70, "smallpond.logical.dataset.DataSet"]], "__init__() (smallpond.logical.dataset.dataset method)": [[70, "smallpond.logical.dataset.DataSet.__init__"]], "fileset (class in smallpond.logical.dataset)": [[71, "smallpond.logical.dataset.FileSet"]], "__init__() (smallpond.logical.dataset.fileset method)": [[71, "smallpond.logical.dataset.FileSet.__init__"]], "jsondataset (class in smallpond.logical.dataset)": [[72, "smallpond.logical.dataset.JsonDataSet"]], "__init__() (smallpond.logical.dataset.jsondataset method)": [[72, "smallpond.logical.dataset.JsonDataSet.__init__"]], "pandasdataset (class in smallpond.logical.dataset)": [[73, "smallpond.logical.dataset.PandasDataSet"]], "__init__() (smallpond.logical.dataset.pandasdataset method)": [[73, "smallpond.logical.dataset.PandasDataSet.__init__"]], "parquetdataset (class in smallpond.logical.dataset)": [[74, "smallpond.logical.dataset.ParquetDataSet"]], "__init__() (smallpond.logical.dataset.parquetdataset method)": [[74, "smallpond.logical.dataset.ParquetDataSet.__init__"]], "partitioneddataset (class in smallpond.logical.dataset)": [[75, "smallpond.logical.dataset.PartitionedDataSet"]], "__init__() (smallpond.logical.dataset.partitioneddataset method)": [[75, "smallpond.logical.dataset.PartitionedDataSet.__init__"]], "sqlquerydataset (class in smallpond.logical.dataset)": [[76, "smallpond.logical.dataset.SqlQueryDataSet"]], "__init__() (smallpond.logical.dataset.sqlquerydataset method)": [[76, "smallpond.logical.dataset.SqlQueryDataSet.__init__"]], "arrowbatchnode (class in smallpond.logical.node)": [[77, "smallpond.logical.node.ArrowBatchNode"]], "__init__() (smallpond.logical.node.arrowbatchnode method)": [[77, "smallpond.logical.node.ArrowBatchNode.__init__"]], "arrowcomputenode (class in smallpond.logical.node)": [[78, "smallpond.logical.node.ArrowComputeNode"]], "__init__() (smallpond.logical.node.arrowcomputenode method)": [[78, "smallpond.logical.node.ArrowComputeNode.__init__"]], "arrowstreamnode (class in smallpond.logical.node)": [[79, "smallpond.logical.node.ArrowStreamNode"]], "__init__() (smallpond.logical.node.arrowstreamnode method)": [[79, "smallpond.logical.node.ArrowStreamNode.__init__"]], "consolidatenode (class in smallpond.logical.node)": [[80, "smallpond.logical.node.ConsolidateNode"]], "__init__() (smallpond.logical.node.consolidatenode method)": [[80, "smallpond.logical.node.ConsolidateNode.__init__"]], "context (class in smallpond.logical.node)": [[81, "smallpond.logical.node.Context"]], "__init__() (smallpond.logical.node.context method)": [[81, "smallpond.logical.node.Context.__init__"]], "datasetpartitionnode() (in module smallpond.logical.node)": [[82, "smallpond.logical.node.DataSetPartitionNode"]], "datasinknode (class in smallpond.logical.node)": [[83, "smallpond.logical.node.DataSinkNode"]], "__init__() (smallpond.logical.node.datasinknode method)": [[83, "smallpond.logical.node.DataSinkNode.__init__"]], "datasourcenode (class in smallpond.logical.node)": [[84, "smallpond.logical.node.DataSourceNode"]], "__init__() (smallpond.logical.node.datasourcenode method)": [[84, "smallpond.logical.node.DataSourceNode.__init__"]], "evenlydistributedpartitionnode (class in smallpond.logical.node)": [[85, "smallpond.logical.node.EvenlyDistributedPartitionNode"]], "__init__() (smallpond.logical.node.evenlydistributedpartitionnode method)": [[85, "smallpond.logical.node.EvenlyDistributedPartitionNode.__init__"]], "hashpartitionnode (class in smallpond.logical.node)": [[86, "smallpond.logical.node.HashPartitionNode"]], "__init__() (smallpond.logical.node.hashpartitionnode method)": [[86, "smallpond.logical.node.HashPartitionNode.__init__"]], "limitnode (class in smallpond.logical.node)": [[87, "smallpond.logical.node.LimitNode"]], "__init__() (smallpond.logical.node.limitnode method)": [[87, "smallpond.logical.node.LimitNode.__init__"]], "loadpartitioneddatasetnode (class in smallpond.logical.node)": [[88, "smallpond.logical.node.LoadPartitionedDataSetNode"]], "__init__() (smallpond.logical.node.loadpartitioneddatasetnode method)": [[88, "smallpond.logical.node.LoadPartitionedDataSetNode.__init__"]], "logicalplan (class in smallpond.logical.node)": [[89, "smallpond.logical.node.LogicalPlan"]], "__init__() (smallpond.logical.node.logicalplan method)": [[89, "smallpond.logical.node.LogicalPlan.__init__"]], "logicalplanvisitor (class in smallpond.logical.node)": [[90, "smallpond.logical.node.LogicalPlanVisitor"]], "__init__() (smallpond.logical.node.logicalplanvisitor method)": [[90, "smallpond.logical.node.LogicalPlanVisitor.__init__"]], "node (class in smallpond.logical.node)": [[91, "smallpond.logical.node.Node"]], "__init__() (smallpond.logical.node.node method)": [[91, "smallpond.logical.node.Node.__init__"]], "nodeid (class in smallpond.logical.node)": [[92, "smallpond.logical.node.NodeId"]], "__init__() (smallpond.logical.node.nodeid method)": [[92, "smallpond.logical.node.NodeId.__init__"]], "pandasbatchnode (class in smallpond.logical.node)": [[93, "smallpond.logical.node.PandasBatchNode"]], "__init__() (smallpond.logical.node.pandasbatchnode method)": [[93, "smallpond.logical.node.PandasBatchNode.__init__"]], "pandascomputenode (class in smallpond.logical.node)": [[94, "smallpond.logical.node.PandasComputeNode"]], "__init__() (smallpond.logical.node.pandascomputenode method)": [[94, "smallpond.logical.node.PandasComputeNode.__init__"]], "partitionnode (class in smallpond.logical.node)": [[95, "smallpond.logical.node.PartitionNode"]], "__init__() (smallpond.logical.node.partitionnode method)": [[95, "smallpond.logical.node.PartitionNode.__init__"]], "projectionnode (class in smallpond.logical.node)": [[96, "smallpond.logical.node.ProjectionNode"]], "__init__() (smallpond.logical.node.projectionnode method)": [[96, "smallpond.logical.node.ProjectionNode.__init__"]], "pythonscriptnode (class in smallpond.logical.node)": [[97, "smallpond.logical.node.PythonScriptNode"]], "__init__() (smallpond.logical.node.pythonscriptnode method)": [[97, "smallpond.logical.node.PythonScriptNode.__init__"]], "rangepartitionnode (class in smallpond.logical.node)": [[98, "smallpond.logical.node.RangePartitionNode"]], "__init__() (smallpond.logical.node.rangepartitionnode method)": [[98, "smallpond.logical.node.RangePartitionNode.__init__"]], "repeatpartitionnode (class in smallpond.logical.node)": [[99, "smallpond.logical.node.RepeatPartitionNode"]], "__init__() (smallpond.logical.node.repeatpartitionnode method)": [[99, "smallpond.logical.node.RepeatPartitionNode.__init__"]], "rootnode (class in smallpond.logical.node)": [[100, "smallpond.logical.node.RootNode"]], "__init__() (smallpond.logical.node.rootnode method)": [[100, "smallpond.logical.node.RootNode.__init__"]], "shufflenode (class in smallpond.logical.node)": [[101, "smallpond.logical.node.ShuffleNode"]], "__init__() (smallpond.logical.node.shufflenode method)": [[101, "smallpond.logical.node.ShuffleNode.__init__"]], "sqlenginenode (class in smallpond.logical.node)": [[102, "smallpond.logical.node.SqlEngineNode"]], "__init__() (smallpond.logical.node.sqlenginenode method)": [[102, "smallpond.logical.node.SqlEngineNode.__init__"]], "unionnode (class in smallpond.logical.node)": [[103, "smallpond.logical.node.UnionNode"]], "__init__() (smallpond.logical.node.unionnode method)": [[103, "smallpond.logical.node.UnionNode.__init__"]], "userdefinedpartitionnode (class in smallpond.logical.node)": [[104, "smallpond.logical.node.UserDefinedPartitionNode"]], "__init__() (smallpond.logical.node.userdefinedpartitionnode method)": [[104, "smallpond.logical.node.UserDefinedPartitionNode.__init__"]], "userpartitioneddatasourcenode (class in smallpond.logical.node)": [[105, "smallpond.logical.node.UserPartitionedDataSourceNode"]], "__init__() (smallpond.logical.node.userpartitioneddatasourcenode method)": [[105, "smallpond.logical.node.UserPartitionedDataSourceNode.__init__"]], "mpi (class in smallpond.platform)": [[106, "smallpond.platform.MPI"]], "__init__() (smallpond.platform.mpi method)": [[106, "smallpond.platform.MPI.__init__"]], "platform (class in smallpond.platform)": [[107, "smallpond.platform.Platform"]], "__init__() (smallpond.platform.platform method)": [[107, "smallpond.platform.Platform.__init__"]]}})