digraph G {
0 [id="node0" labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>" tooltip="AdaptiveSparkPlan isFinalPlan=true"];
subgraph cluster1 {
isCluster="true";
id="cluster1";
label="WholeStageCodegen (6)\n \nduration: 4 ms";
tooltip="WholeStageCodegen (6)";
2 [id="node2" labelType="html" label="<b>HashAggregate</b><br><br>time in aggregation build: 3 ms<br>number of output rows: 1" tooltip="HashAggregate(keys=[], functions=[count(1)])"];
}
3 [id="node3" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 37<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 37<br>local bytes read: 1500.0 B<br>merged fetch fallback count: 0<br>local blocks read: 20<br>remote merged chunks fetched: 0<br>remote blocks read: 17<br>data size total (min, med, max (stageId: taskId))<br>592.0 B (16.0 B, 16.0 B, 16.0 B (stage 126.0: task 902))<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>18 ms (0 ms, 0 ms, 11 ms (stage 126.0: task 901))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 1275.0 B<br>number of partitions: 1<br>remote reqs duration: 1 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>2.7 KiB (75.0 B, 75.0 B, 75.0 B (stage 126.0: task 902))" tooltip="Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2349]"];
subgraph cluster4 {
isCluster="true";
id="cluster4";
label="WholeStageCodegen (5)\n \nduration: total (min, med, max (stageId: taskId))\n23.0 s (440 ms, 553 ms, 939 ms (stage 126.0: task 895))";
tooltip="WholeStageCodegen (5)";
5 [id="node5" labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build total (min, med, max (stageId: taskId))<br>22.9 s (440 ms, 552 ms, 937 ms (stage 126.0: task 895))<br>peak memory: 0.0 B<br>number of output rows: 37<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0" tooltip="HashAggregate(keys=[], functions=[partial_count(1)])"];
6 [id="node6" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project"];
7 [id="node7" labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 9,999<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 126.0: task 902))" tooltip="SortMergeJoin [product#1127, plant#1128], [product#2322, plant#2323], Inner"];
}
subgraph cluster8 {
isCluster="true";
id="cluster8";
label="WholeStageCodegen (3)\n \nduration: total (min, med, max (stageId: taskId))\n25.7 s (461 ms, 556 ms, 1.2 s (stage 126.0: task 895))";
tooltip="WholeStageCodegen (3)";
9 [id="node9" labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 126.0: task 902))<br>peak memory total (min, med, max (stageId: taskId))<br>76.3 MiB (2.1 MiB, 2.1 MiB, 2.1 MiB (stage 126.0: task 902))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 126.0: task 902))" tooltip="Sort [product#1127 ASC NULLS FIRST, plant#1128 ASC NULLS FIRST], false, 0"];
}
10 [id="node10" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 9,999<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 9,999<br>local bytes read total (min, med, max (stageId: taskId))<br>226.9 KiB (3.0 KiB, 3.9 KiB, 10.4 KiB (stage 126.0: task 899))<br>merged fetch fallback count: 0<br>local blocks read: 530<br>remote merged chunks fetched: 0<br>remote blocks read: 749<br>data size total (min, med, max (stageId: taskId))<br>859.3 KiB (8.5 KiB, 25.5 KiB, 68.1 KiB (stage 110.0: task 741))<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>73 ms (1 ms, 2 ms, 2 ms (stage 110.0: task 813))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time total (min, med, max (stageId: taskId))<br>242 ms (0 ms, 0 ms, 54 ms (stage 126.0: task 913))<br>remote bytes read total (min, med, max (stageId: taskId))<br>328.2 KiB (4.2 KiB, 10.9 KiB, 12.8 KiB (stage 126.0: task 904))<br>number of partitions: 37<br>remote reqs duration total (min, med, max (stageId: taskId))<br>718 ms (4 ms, 6 ms, 91 ms (stage 126.0: task 913))<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>555.1 KiB (7.0 KiB, 16.4 KiB, 38.2 KiB (stage 110.0: task 741))" tooltip="Exchange hashpartitioning(product#1127, plant#1128, 37), ENSURE_REQUIREMENTS, [plan_id=2218]"];
subgraph cluster11 {
isCluster="true";
id="cluster11";
label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n1.1 s (0 ms, 8 ms, 103 ms (stage 110.0: task 735))";
tooltip="WholeStageCodegen (2)";
12 [id="node12" labelType="html" label="<b>Filter</b><br><br>number of output rows: 9,999" tooltip="Filter (isnotnull(product#1127) AND isnotnull(plant#1128))"];
}
13 [id="node13" labelType="html" label="<b>InMemoryTableScan</b><br><br>number of output rows: 9,999" tooltip="InMemoryTableScan [product#1127, plant#1128], [isnotnull(product#1127), isnotnull(plant#1128)]"];
14 [id="node14" labelType="html" label="<br><b>AdaptiveSparkPlan</b><br><br>" tooltip="AdaptiveSparkPlan isFinalPlan=true"];
subgraph cluster15 {
isCluster="true";
id="cluster15";
label="WholeStageCodegen (12)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (12)";
16 [id="node16" labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 0.0 B<br>number of output rows: 0<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0" tooltip="HashAggregate(keys=[product#1127, plant#1128], functions=[])"];
17 [id="node17" labelType="html" label="<b>HashAggregate</b><br><br>spill size: 0.0 B<br>time in aggregation build: 0 ms<br>peak memory: 0.0 B<br>number of output rows: 0<br>number of sort fallback tasks: 0<br>avg hash probes per key: 0" tooltip="HashAggregate(keys=[product#1127, plant#1128], functions=[])"];
18 [id="node18" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#1127, plant#1128]"];
19 [id="node19" labelType="html" label="<b>Filter</b><br><br>number of output rows: 0" tooltip="Filter (((size(demandTimeBuckets#1577, true) = 0) OR isnull(demandTimeBuckets#1577)) AND ((size(demandPointInTime#1580, true) = 0) OR isnull(demandPointInTime#1580)))"];
20 [id="node20" labelType="html" label="<b>BroadcastHashJoin</b><br><br>number of output rows: 0" tooltip="BroadcastHashJoin [coalesce(plant#1128, ), isnull(plant#1128), coalesce(product#1127, ), isnull(product#1127)], [coalesce(plant#1573, ), isnull(plant#1573), coalesce(product#1572, ), isnull(product#1572)], LeftOuter, BuildRight, false"];
21 [id="node21" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#1127, plant#1128]"];
22 [id="node22" labelType="html" label="<b>Generate</b><br><br>number of output rows: 0" tooltip="Generate explode(_extract_quantityDenominator#1607), [product#1127, plant#1128], true, [quantityDenominator#1216]"];
23 [id="node23" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205.quantityDenominator AS _extract_quantityDenominator#1607]"];
24 [id="node24" labelType="html" label="<b>Generate</b><br><br>number of output rows: 0" tooltip="Generate explode(filteredUnitOfMeasureConversions#1205.quantityNumerator), [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205], true, [quantityNumerator#1213]"];
}
25 [id="node25" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#1127, plant#1128, filter(unitOfMeasureConversions#1200, lambdafunction((((lambda u#1206.measurementUnit1.code = _extract_code#1686) AND (lambda u#1206.measurementUnit1.internalRefUUID = _extract_internalRefUUID#1687)) AND ((lambda u#1206.measurementUnit2.code = baseUnitOfMeasure#1198.code) AND (lambda u#1206.measurementUnit2.internalRefUUID = baseUnitOfMeasure#1198.internalRefUUID))), lambda u#1206, false)) AS filteredUnitOfMeasureConversions#1205]"];
subgraph cluster26 {
isCluster="true";
id="cluster26";
label="WholeStageCodegen (11)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (11)";
27 [id="node27" labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 0<br>spill size: 0.0 B" tooltip="SortMergeJoin [product#1127], [internalUUID#1194], LeftOuter"];
}
subgraph cluster28 {
isCluster="true";
id="cluster28";
label="WholeStageCodegen (9)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (9)";
29 [id="node29" labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 0.0 B<br>spill size: 0.0 B" tooltip="Sort [product#1127 ASC NULLS FIRST], false, 0"];
}
30 [id="node30" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(product#1127, 37), ENSURE_REQUIREMENTS, [plan_id=1895]"];
subgraph cluster31 {
isCluster="true";
id="cluster31";
label="WholeStageCodegen (8)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (8)";
32 [id="node32" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]"];
33 [id="node33" labelType="html" label="<b>SortMergeJoin</b><br><br>number of output rows: 0<br>spill size: 0.0 B" tooltip="SortMergeJoin [coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128)], [coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689)], LeftOuter"];
}
subgraph cluster34 {
isCluster="true";
id="cluster34";
label="WholeStageCodegen (6)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (6)";
35 [id="node35" labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 0.0 B<br>spill size: 0.0 B" tooltip="Sort [coalesce(product#1127, ) ASC NULLS FIRST, isnull(product#1127) ASC NULLS FIRST, coalesce(plant#1128, ) ASC NULLS FIRST, isnull(plant#1128) ASC NULLS FIRST], false, 0"];
}
36 [id="node36" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128), 37), ENSURE_REQUIREMENTS, [plan_id=1225]"];
37 [id="node37" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(product#1127, plant#1128, 37), REPARTITION_BY_NUM, [plan_id=1218]"];
subgraph cluster38 {
isCluster="true";
id="cluster38";
label="WholeStageCodegen (1)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (1)";
39 [id="node39" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 0<br>number of input batches: 0" tooltip="ColumnarToRow"];
}
40 [id="node40" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 0<br>scan time: 0 ms<br>metadata time: 0 ms<br>size of files read: 0.0 B<br>number of output rows: 0" tooltip="FileScan parquet [product#1127,plant#1128] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<product:string,plant:string>"];
subgraph cluster41 {
isCluster="true";
id="cluster41";
label="WholeStageCodegen (7)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (7)";
42 [id="node42" labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 0.0 B<br>spill size: 0.0 B" tooltip="Sort [coalesce(_extract_internalRefUUID#1688, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1688) ASC NULLS FIRST, coalesce(_extract_internalRefUUID#1689, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1689) ASC NULLS FIRST], false, 0"];
}
43 [id="node43" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689), 37), ENSURE_REQUIREMENTS, [plan_id=1786]"];
subgraph cluster44 {
isCluster="true";
id="cluster44";
label="WholeStageCodegen (5)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (5)";
45 [id="node45" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]"];
}
46 [id="node46" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1234]"];
subgraph cluster47 {
isCluster="true";
id="cluster47";
label="WholeStageCodegen (2)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (2)";
48 [id="node48" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [plant#1160.internalRefUUID AS _extract_internalRefUUID#1689, product#1161.internalRefUUID AS _extract_internalRefUUID#1688, productionAspect#1163.productMovementPlants.unitOfIssue.code AS _extract_code#1686, productionAspect#1163.productMovementPlants.unitOfIssue.internalRefUUID AS _extract_internalRefUUID#1687]"];
49 [id="node49" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 0<br>number of input batches: 0" tooltip="ColumnarToRow"];
}
50 [id="node50" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 0<br>scan time: 0 ms<br>dynamic partition pruning time: 0 ms<br>metadata time: 0 ms<br>size of files read: 0.0 B<br>number of output rows: 0<br>number of partitions read: 0" tooltip="FileScan parquet [plant#1160,product#1161,productionAspect#1163,systemId#1158] Batched: true, DataFilters: [], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<plant:struct<internalRefUUID:string>,product:struct<internalRefUUID:string>,productionAspe..."];
subgraph cluster51 {
isCluster="true";
id="cluster51";
label="WholeStageCodegen (10)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (10)";
52 [id="node52" labelType="html" label="<b>Sort</b><br><br>sort time: 0 ms<br>peak memory: 0.0 B<br>spill size: 0.0 B" tooltip="Sort [internalUUID#1194 ASC NULLS FIRST], false, 0"];
}
53 [id="node53" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(internalUUID#1194, 37), ENSURE_REQUIREMENTS, [plan_id=1276]"];
54 [id="node54" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 0<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 0<br>local bytes read: 0.0 B<br>merged fetch fallback count: 0<br>local blocks read: 0<br>remote merged chunks fetched: 0<br>remote blocks read: 0<br>data size: 0.0 B<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time: 0 ms<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time: 0 ms<br>remote bytes read: 0.0 B<br>number of partitions: 0<br>remote reqs duration: 0 ms<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written: 0.0 B" tooltip="Exchange RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1266]"];
subgraph cluster55 {
isCluster="true";
id="cluster55";
label="WholeStageCodegen (3)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (3)";
56 [id="node56" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]"];
57 [id="node57" labelType="html" label="<b>Filter</b><br><br>number of output rows: 0" tooltip="Filter isnotnull(internalUUID#1194)"];
58 [id="node58" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 0<br>number of input batches: 0" tooltip="ColumnarToRow"];
}
59 [id="node59" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 0<br>scan time: 0 ms<br>dynamic partition pruning time: 0 ms<br>metadata time: 0 ms<br>size of files read: 0.0 B<br>number of output rows: 0<br>number of partitions read: 0" tooltip="FileScan parquet [internalUUID#1194,baseUnitOfMeasure#1198,unitOfMeasureConversions#1200,systemId#1193] Batched: true, DataFilters: [isnotnull(internalUUID#1194)], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [IsNotNull(internalUUID)], ReadSchema: struct<internalUUID:string,baseUnitOfMeasure:struct<internalRefUUID:string,code:string>,unitOfMea..."];
60 [id="node60" labelType="html" label="<b>BroadcastExchange</b><br><br>time to broadcast: 0 ms<br>time to build: 0 ms<br>time to collect: 0 ms<br>number of output rows: 0<br>data size: 0.0 B" tooltip="BroadcastExchange HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1291]"];
subgraph cluster61 {
isCluster="true";
id="cluster61";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (4)";
62 [id="node62" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 0<br>number of input batches: 0" tooltip="ColumnarToRow"];
}
63 [id="node63" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 0<br>scan time: 0 ms<br>metadata time: 0 ms<br>size of files read: 0.0 B<br>number of output rows: 0" tooltip="FileScan parquet [product#1572,plant#1573,demandTimeBuckets#1577,demandPointInTime#1580] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanac..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<product:string,plant:string,demandTimeBuckets:array<struct<demandStartDateTime:timestamp,d..."];
subgraph cluster64 {
isCluster="true";
id="cluster64";
label="WholeStageCodegen (4)\n \nduration: 0 ms";
tooltip="WholeStageCodegen (4)";
65 [id="node65" labelType="html" label="<b>Sort</b><br><br>sort time total (min, med, max (stageId: taskId))<br>9.3 s (217 ms, 245 ms, 321 ms (stage 126.0: task 915))<br>peak memory total (min, med, max (stageId: taskId))<br>1998.0 MiB (54.0 MiB, 54.0 MiB, 54.0 MiB (stage 126.0: task 902))<br>spill size total (min, med, max (stageId: taskId))<br>0.0 B (0.0 B, 0.0 B, 0.0 B (stage 126.0: task 902))" tooltip="Sort [product#2322 ASC NULLS FIRST, plant#2323 ASC NULLS FIRST], false, 0"];
}
66 [id="node66" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 15,976,839<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 15,976,839<br>local bytes read total (min, med, max (stageId: taskId))<br>221.2 MiB (4.9 MiB, 6.9 MiB, 6.9 MiB (stage 126.0: task 906))<br>merged fetch fallback count: 0<br>local blocks read: 450<br>remote merged chunks fetched: 0<br>remote blocks read: 919<br>data size total (min, med, max (stageId: taskId))<br>1340.8 MiB (36.2 MiB, 36.2 MiB, 36.2 MiB (stage 115.0: task 869))<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>4.7 s (23 ms, 28 ms, 3.4 s (stage 115.0: task 886))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time total (min, med, max (stageId: taskId))<br>0 ms (0 ms, 0 ms, 0 ms (stage 126.0: task 902))<br>remote bytes read total (min, med, max (stageId: taskId))<br>451.5 MiB (11.3 MiB, 11.3 MiB, 13.3 MiB (stage 126.0: task 903))<br>number of partitions: 37<br>remote reqs duration total (min, med, max (stageId: taskId))<br>2.4 s (27 ms, 58 ms, 123 ms (stage 126.0: task 913))<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>672.6 MiB (18.2 MiB, 18.2 MiB, 18.2 MiB (stage 115.0: task 858))" tooltip="Exchange hashpartitioning(product#2322, plant#2323, 37), ENSURE_REQUIREMENTS, [plan_id=2167]"];
67 [id="node67" labelType="html" label="<b>Exchange</b><br><br>shuffle records written: 15,976,839<br>remote merged reqs duration: 0 ms<br>remote merged blocks fetched: 0<br>records read: 15,976,839<br>local bytes read total (min, med, max (stageId: taskId))<br>210.4 MiB (4.1 MiB, 4.1 MiB, 9.9 MiB (stage 115.0: task 868))<br>merged fetch fallback count: 0<br>local blocks read: 104<br>remote merged chunks fetched: 0<br>remote blocks read: 229<br>data size total (min, med, max (stageId: taskId))<br>1340.8 MiB (125.6 MiB, 151.9 MiB, 151.9 MiB (stage 113.0: task 830))<br>local merged bytes read: 0.0 B<br>local merged chunks fetched: 0<br>shuffle write time total (min, med, max (stageId: taskId))<br>1.2 s (76 ms, 90 ms, 466 ms (stage 113.0: task 838))<br>remote merged bytes read: 0.0 B<br>local merged blocks fetched: 0<br>corrupt merged block chunks: 0<br>fetch wait time total (min, med, max (stageId: taskId))<br>26 ms (0 ms, 0 ms, 11 ms (stage 115.0: task 863))<br>remote bytes read total (min, med, max (stageId: taskId))<br>461.6 MiB (8.2 MiB, 14.0 MiB, 14.1 MiB (stage 115.0: task 889))<br>number of partitions: 37<br>remote reqs duration total (min, med, max (stageId: taskId))<br>4.7 s (25 ms, 107 ms, 267 ms (stage 115.0: task 870))<br>remote bytes read to disk: 0.0 B<br>shuffle bytes written total (min, med, max (stageId: taskId))<br>672.0 MiB (62.9 MiB, 76.1 MiB, 76.1 MiB (stage 113.0: task 838))" tooltip="Exchange RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=2157]"];
subgraph cluster68 {
isCluster="true";
id="cluster68";
label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n19.7 s (58 ms, 129 ms, 2.3 s (stage 113.0: task 838))";
tooltip="WholeStageCodegen (1)";
69 [id="node69" labelType="html" label="<br><b>Project</b><br><br>" tooltip="Project [product#2316.internalRefUUID AS product#2322, plant#2315.internalRefUUID AS plant#2323]"];
70 [id="node70" labelType="html" label="<b>Filter</b><br><br>number of output rows: 15,976,839" tooltip="Filter (isnotnull(product#2316.internalRefUUID) AND isnotnull(plant#2315.internalRefUUID))"];
71 [id="node71" labelType="html" label="<b>ColumnarToRow</b><br><br>number of output rows: 15,976,839<br>number of input batches: 3,902" tooltip="ColumnarToRow"];
}
72 [id="node72" labelType="html" label="<b>Scan parquet </b><br><br>number of files read: 1<br>scan time total (min, med, max (stageId: taskId))<br>15.1 s (58 ms, 128 ms, 1.8 s (stage 113.0: task 838))<br>dynamic partition pruning time: 0 ms<br>metadata time: 417 ms<br>size of files read: 1123.8 MiB<br>number of output rows: 15,976,839<br>number of partitions read: 1" tooltip="FileScan parquet [plant#2315,product#2316,systemId#2313] Batched: true, DataFilters: [isnotnull(product#2316.internalRefUUID), isnotnull(plant#2315.internalRefUUID)], Format: Parquet, Location: PreparedDeltaFileIndex(1 paths)[hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20...., PartitionFilters: [], PushedFilters: [IsNotNull(product.internalRefUUID), IsNotNull(plant.internalRefUUID)], ReadSchema: struct<plant:struct<internalRefUUID:string>,product:struct<internalRefUUID:string>>"];
2->0;
3->2;
5->3;
6->5;
7->6;
9->7;
10->9;
12->10;
13->12;
14->13;
16->14;
17->16;
18->17;
19->18;
20->19;
21->20;
22->21;
23->22;
24->23;
25->24;
27->25;
29->27;
30->29;
32->30;
33->32;
35->33;
36->35;
37->36;
39->37;
40->39;
42->33;
43->42;
45->43;
46->45;
48->46;
49->48;
50->49;
52->27;
53->52;
54->53;
56->54;
57->56;
58->57;
59->58;
60->20;
62->60;
63->62;
65->7;
66->65;
67->66;
69->67;
70->69;
71->70;
72->71;
}
== Physical Plan ==
AdaptiveSparkPlan (113)
+- == Final Plan ==
ResultQueryStage (99), Statistics(sizeInBytes=8.0 EiB)
+- * HashAggregate (98)
+- ShuffleQueryStage (97), Statistics(sizeInBytes=592.0 B, rowCount=37)
+- Exchange (96)
+- * HashAggregate (95)
+- * Project (94)
+- * SortMergeJoin Inner (93)
:- * Sort (83)
: +- ShuffleQueryStage (82), Statistics(sizeInBytes=859.3 KiB, rowCount=1.00E+4)
: +- Exchange (81)
: +- * Filter (80)
: +- TableCacheQueryStage (79), Statistics(sizeInBytes=703.1 KiB, rowCount=1.00E+4)
: +- InMemoryTableScan (1)
: +- InMemoryRelation (2)
: +- AdaptiveSparkPlan (78)
+- == Final Plan ==
ResultQueryStage (48)
+- * HashAggregate (47)
+- * HashAggregate (46)
+- * Project (45)
+- * Filter (44)
+- * BroadcastHashJoin LeftOuter BuildRight (43)
:- * Project (38)
: +- * Generate (37)
: +- * Project (36)
: +- * Generate (35)
: +- Project (34)
: +- * SortMergeJoin LeftOuter (33)
: :- * Sort (23)
: : +- ShuffleQueryStage (22), Statistics(sizeInBytes=1406.1 KiB, rowCount=1.00E+4)
: : +- Exchange (21)
: : +- * Project (20)
: : +- * SortMergeJoin LeftOuter (19)
: : :- * Sort (9)
: : : +- ShuffleQueryStage (8), Statistics(sizeInBytes=859.3 KiB, rowCount=1.00E+4)
: : : +- Exchange (7)
: : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=859.3 KiB, rowCount=1.00E+4)
: : : +- Exchange (5)
: : : +- * ColumnarToRow (4)
: : : +- Scan parquet (3)
: : +- * Sort (18)
: : +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.1 GiB, rowCount=1.60E+7)
: : +- Exchange (16)
: : +- * Project (15)
: : +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.1 GiB, rowCount=1.60E+7)
: : +- Exchange (13)
: : +- * Project (12)
: : +- * ColumnarToRow (11)
: : +- Scan parquet (10)
: +- * Sort (32)
: +- ShuffleQueryStage (31), Statistics(sizeInBytes=73.4 MiB, rowCount=1.55E+5)
: +- Exchange (30)
: +- ShuffleQueryStage (29), Statistics(sizeInBytes=73.4 MiB, rowCount=1.55E+5)
: +- Exchange (28)
: +- * Project (27)
: +- * Filter (26)
: +- * ColumnarToRow (25)
: +- Scan parquet (24)
+- BroadcastQueryStage (42), Statistics(sizeInBytes=4.5 MiB, rowCount=1.00E+4)
+- BroadcastExchange (41)
+- * ColumnarToRow (40)
+- Scan parquet (39)
+- == Initial Plan ==
HashAggregate (77)
+- HashAggregate (76)
+- Project (75)
+- Filter (74)
+- BroadcastHashJoin LeftOuter BuildRight (73)
:- Project (71)
: +- Generate (70)
: +- Project (69)
: +- Generate (68)
: +- Project (67)
: +- SortMergeJoin LeftOuter (66)
: :- Sort (60)
: : +- Exchange (59)
: : +- Project (58)
: : +- SortMergeJoin LeftOuter (57)
: : :- Sort (51)
: : : +- Exchange (50)
: : : +- Exchange (49)
: : : +- Scan parquet (3)
: : +- Sort (56)
: : +- Exchange (55)
: : +- Project (54)
: : +- Exchange (53)
: : +- Project (52)
: : +- Scan parquet (10)
: +- Sort (65)
: +- Exchange (64)
: +- Exchange (63)
: +- Project (62)
: +- Filter (61)
: +- Scan parquet (24)
+- BroadcastExchange (72)
+- Scan parquet (39)
+- * Sort (92)
+- ShuffleQueryStage (91), Statistics(sizeInBytes=1340.8 MiB, rowCount=1.60E+7)
+- Exchange (90)
+- ShuffleQueryStage (89), Statistics(sizeInBytes=1340.8 MiB, rowCount=1.60E+7)
+- Exchange (88)
+- * Project (87)
+- * Filter (86)
+- * ColumnarToRow (85)
+- Scan parquet (84)
+- == Initial Plan ==
HashAggregate (112)
+- Exchange (111)
+- HashAggregate (110)
+- Project (109)
+- SortMergeJoin Inner (108)
:- Sort (102)
: +- Exchange (101)
: +- Filter (100)
: +- InMemoryTableScan (1)
: +- InMemoryRelation (2)
: +- AdaptiveSparkPlan (78)
+- == Final Plan ==
ResultQueryStage (48)
+- * HashAggregate (47)
+- * HashAggregate (46)
+- * Project (45)
+- * Filter (44)
+- * BroadcastHashJoin LeftOuter BuildRight (43)
:- * Project (38)
: +- * Generate (37)
: +- * Project (36)
: +- * Generate (35)
: +- Project (34)
: +- * SortMergeJoin LeftOuter (33)
: :- * Sort (23)
: : +- ShuffleQueryStage (22), Statistics(sizeInBytes=1406.1 KiB, rowCount=1.00E+4)
: : +- Exchange (21)
: : +- * Project (20)
: : +- * SortMergeJoin LeftOuter (19)
: : :- * Sort (9)
: : : +- ShuffleQueryStage (8), Statistics(sizeInBytes=859.3 KiB, rowCount=1.00E+4)
: : : +- Exchange (7)
: : : +- ShuffleQueryStage (6), Statistics(sizeInBytes=859.3 KiB, rowCount=1.00E+4)
: : : +- Exchange (5)
: : : +- * ColumnarToRow (4)
: : : +- Scan parquet (3)
: : +- * Sort (18)
: : +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.1 GiB, rowCount=1.60E+7)
: : +- Exchange (16)
: : +- * Project (15)
: : +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.1 GiB, rowCount=1.60E+7)
: : +- Exchange (13)
: : +- * Project (12)
: : +- * ColumnarToRow (11)
: : +- Scan parquet (10)
: +- * Sort (32)
: +- ShuffleQueryStage (31), Statistics(sizeInBytes=73.4 MiB, rowCount=1.55E+5)
: +- Exchange (30)
: +- ShuffleQueryStage (29), Statistics(sizeInBytes=73.4 MiB, rowCount=1.55E+5)
: +- Exchange (28)
: +- * Project (27)
: +- * Filter (26)
: +- * ColumnarToRow (25)
: +- Scan parquet (24)
+- BroadcastQueryStage (42), Statistics(sizeInBytes=4.5 MiB, rowCount=1.00E+4)
+- BroadcastExchange (41)
+- * ColumnarToRow (40)
+- Scan parquet (39)
+- == Initial Plan ==
HashAggregate (77)
+- HashAggregate (76)
+- Project (75)
+- Filter (74)
+- BroadcastHashJoin LeftOuter BuildRight (73)
:- Project (71)
: +- Generate (70)
: +- Project (69)
: +- Generate (68)
: +- Project (67)
: +- SortMergeJoin LeftOuter (66)
: :- Sort (60)
: : +- Exchange (59)
: : +- Project (58)
: : +- SortMergeJoin LeftOuter (57)
: : :- Sort (51)
: : : +- Exchange (50)
: : : +- Exchange (49)
: : : +- Scan parquet (3)
: : +- Sort (56)
: : +- Exchange (55)
: : +- Project (54)
: : +- Exchange (53)
: : +- Project (52)
: : +- Scan parquet (10)
: +- Sort (65)
: +- Exchange (64)
: +- Exchange (63)
: +- Project (62)
: +- Filter (61)
: +- Scan parquet (24)
+- BroadcastExchange (72)
+- Scan parquet (39)
+- Sort (107)
+- Exchange (106)
+- Exchange (105)
+- Project (104)
+- Filter (103)
+- Scan parquet (84)
(1) InMemoryTableScan
Output [2]: [product#1127, plant#1128]
Arguments: [product#1127, plant#1128], [isnotnull(product#1127), isnotnull(plant#1128)]
(2) InMemoryRelation
Arguments: [product#1127, plant#1128], StorageLevel(disk, memory, 2 replicas)
(3) Scan parquet
Output [2]: [product#1127, plant#1128]
Batched: true
Location: InMemoryFileIndex [hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-order-qty-opt-service/out/product-plant-list/10000000004/shardId=0_1_10000000004]
ReadSchema: struct<product:string,plant:string>
(4) ColumnarToRow [codegen id : 1]
Input [2]: [product#1127, plant#1128]
(5) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(product#1127, plant#1128, 37), REPARTITION_BY_NUM, [plan_id=1218]
(6) ShuffleQueryStage
Output [2]: [product#1127, plant#1128]
Arguments: 0
(7) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128), 37), ENSURE_REQUIREMENTS, [plan_id=1225]
(8) ShuffleQueryStage
Output [2]: [product#1127, plant#1128]
Arguments: 4
(9) Sort [codegen id : 6]
Input [2]: [product#1127, plant#1128]
Arguments: [coalesce(product#1127, ) ASC NULLS FIRST, isnull(product#1127) ASC NULLS FIRST, coalesce(plant#1128, ) ASC NULLS FIRST, isnull(plant#1128) ASC NULLS FIRST], false, 0
(10) Scan parquet
Output [4]: [plant#1160, product#1161, productionAspect#1163, systemId#1158]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant]
ReadSchema: struct<plant:struct<internalRefUUID:string>,product:struct<internalRefUUID:string>,productionAspect:struct<productMovementPlants:struct<unitOfIssue:struct<internalRefUUID:string,code:string>>>>
(11) ColumnarToRow [codegen id : 2]
Input [4]: [plant#1160, product#1161, productionAspect#1163, systemId#1158]
(12) Project [codegen id : 2]
Output [4]: [plant#1160.internalRefUUID AS _extract_internalRefUUID#1689, product#1161.internalRefUUID AS _extract_internalRefUUID#1688, productionAspect#1163.productMovementPlants.unitOfIssue.code AS _extract_code#1686, productionAspect#1163.productMovementPlants.unitOfIssue.internalRefUUID AS _extract_internalRefUUID#1687]
Input [4]: [plant#1160, product#1161, productionAspect#1163, systemId#1158]
(13) Exchange
Input [4]: [_extract_internalRefUUID#1689, _extract_internalRefUUID#1688, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1234]
(14) ShuffleQueryStage
Output [4]: [_extract_internalRefUUID#1689, _extract_internalRefUUID#1688, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: 1
(15) Project [codegen id : 5]
Output [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Input [4]: [_extract_internalRefUUID#1689, _extract_internalRefUUID#1688, _extract_code#1686, _extract_internalRefUUID#1687]
(16) Exchange
Input [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Arguments: hashpartitioning(coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689), 37), ENSURE_REQUIREMENTS, [plan_id=1786]
(17) ShuffleQueryStage
Output [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Arguments: 6
(18) Sort [codegen id : 7]
Input [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Arguments: [coalesce(_extract_internalRefUUID#1688, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1688) ASC NULLS FIRST, coalesce(_extract_internalRefUUID#1689, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1689) ASC NULLS FIRST], false, 0
(19) SortMergeJoin [codegen id : 8]
Left keys [4]: [coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128)]
Right keys [4]: [coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689)]
Join type: LeftOuter
Join condition: None
(20) Project [codegen id : 8]
Output [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Input [6]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
(21) Exchange
Input [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: hashpartitioning(product#1127, 37), ENSURE_REQUIREMENTS, [plan_id=1895]
(22) ShuffleQueryStage
Output [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: 7
(23) Sort [codegen id : 9]
Input [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: [product#1127 ASC NULLS FIRST], false, 0
(24) Scan parquet
Output [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-product-v2]
PushedFilters: [IsNotNull(internalUUID)]
ReadSchema: struct<internalUUID:string,baseUnitOfMeasure:struct<internalRefUUID:string,code:string>,unitOfMeasureConversions:array<struct<measurementUnit1:struct<internalRefUUID:string,code:string>,measurementUnit2:struct<internalRefUUID:string,code:string>,quantityNumerator:int,quantityDenominator:int>>>
(25) ColumnarToRow [codegen id : 3]
Input [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
(26) Filter [codegen id : 3]
Input [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
Condition : isnotnull(internalUUID#1194)
(27) Project [codegen id : 3]
Output [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Input [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
(28) Exchange
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1266]
(29) ShuffleQueryStage
Output [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: 2
(30) Exchange
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: hashpartitioning(internalUUID#1194, 37), ENSURE_REQUIREMENTS, [plan_id=1276]
(31) ShuffleQueryStage
Output [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: 5
(32) Sort [codegen id : 10]
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: [internalUUID#1194 ASC NULLS FIRST], false, 0
(33) SortMergeJoin [codegen id : 11]
Left keys [1]: [product#1127]
Right keys [1]: [internalUUID#1194]
Join type: LeftOuter
Join condition: None
(34) Project
Output [3]: [product#1127, plant#1128, filter(unitOfMeasureConversions#1200, lambdafunction((((lambda u#1206.measurementUnit1.code = _extract_code#1686) AND (lambda u#1206.measurementUnit1.internalRefUUID = _extract_internalRefUUID#1687)) AND ((lambda u#1206.measurementUnit2.code = baseUnitOfMeasure#1198.code) AND (lambda u#1206.measurementUnit2.internalRefUUID = baseUnitOfMeasure#1198.internalRefUUID))), lambda u#1206, false)) AS filteredUnitOfMeasureConversions#1205]
Input [7]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687, internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
(35) Generate [codegen id : 12]
Input [3]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205]
Arguments: explode(filteredUnitOfMeasureConversions#1205.quantityNumerator), [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205], true, [quantityNumerator#1213]
(36) Project [codegen id : 12]
Output [3]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205.quantityDenominator AS _extract_quantityDenominator#1607]
Input [4]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205, quantityNumerator#1213]
(37) Generate [codegen id : 12]
Input [3]: [product#1127, plant#1128, _extract_quantityDenominator#1607]
Arguments: explode(_extract_quantityDenominator#1607), [product#1127, plant#1128], true, [quantityDenominator#1216]
(38) Project [codegen id : 12]
Output [2]: [product#1127, plant#1128]
Input [3]: [product#1127, plant#1128, quantityDenominator#1216]
(39) Scan parquet
Output [4]: [product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Batched: true
Location: InMemoryFileIndex [hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-demand-service/out/demand/10000000004/0_1_10000000004]
ReadSchema: struct<product:string,plant:string,demandTimeBuckets:array<struct<demandStartDateTime:timestamp,demandQuantity:double,demandVariance:double>>,demandPointInTime:array<struct<demandDateTime:timestamp,demandQuantity:double,demandVariance:double>>>
(40) ColumnarToRow [codegen id : 4]
Input [4]: [product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
(41) BroadcastExchange
Input [4]: [product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1291]
(42) BroadcastQueryStage
Output [4]: [product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Arguments: 3
(43) BroadcastHashJoin [codegen id : 12]
Left keys [4]: [coalesce(plant#1128, ), isnull(plant#1128), coalesce(product#1127, ), isnull(product#1127)]
Right keys [4]: [coalesce(plant#1573, ), isnull(plant#1573), coalesce(product#1572, ), isnull(product#1572)]
Join type: LeftOuter
Join condition: None
(44) Filter [codegen id : 12]
Input [6]: [product#1127, plant#1128, product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Condition : (((size(demandTimeBuckets#1577, true) = 0) OR isnull(demandTimeBuckets#1577)) AND ((size(demandPointInTime#1580, true) = 0) OR isnull(demandPointInTime#1580)))
(45) Project [codegen id : 12]
Output [2]: [product#1127, plant#1128]
Input [6]: [product#1127, plant#1128, product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
(46) HashAggregate [codegen id : 12]
Input [2]: [product#1127, plant#1128]
Keys [2]: [product#1127, plant#1128]
Functions: []
Aggregate Attributes: []
Results [2]: [product#1127, plant#1128]
(47) HashAggregate [codegen id : 12]
Input [2]: [product#1127, plant#1128]
Keys [2]: [product#1127, plant#1128]
Functions: []
Aggregate Attributes: []
Results [2]: [product#1127, plant#1128]
(48) ResultQueryStage
Output [2]: [product#1127, plant#1128]
Arguments: 8
(49) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(product#1127, plant#1128, 37), REPARTITION_BY_NUM, [plan_id=1131]
(50) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128), 37), ENSURE_REQUIREMENTS, [plan_id=1156]
(51) Sort
Input [2]: [product#1127, plant#1128]
Arguments: [coalesce(product#1127, ) ASC NULLS FIRST, isnull(product#1127) ASC NULLS FIRST, coalesce(plant#1128, ) ASC NULLS FIRST, isnull(plant#1128) ASC NULLS FIRST], false, 0
(52) Project
Output [4]: [plant#1160.internalRefUUID AS _extract_internalRefUUID#1689, product#1161.internalRefUUID AS _extract_internalRefUUID#1688, productionAspect#1163.productMovementPlants.unitOfIssue.code AS _extract_code#1686, productionAspect#1163.productMovementPlants.unitOfIssue.internalRefUUID AS _extract_internalRefUUID#1687]
Input [4]: [plant#1160, product#1161, productionAspect#1163, systemId#1158]
(53) Exchange
Input [4]: [_extract_internalRefUUID#1689, _extract_internalRefUUID#1688, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1134]
(54) Project
Output [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Input [4]: [_extract_internalRefUUID#1689, _extract_internalRefUUID#1688, _extract_code#1686, _extract_internalRefUUID#1687]
(55) Exchange
Input [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Arguments: hashpartitioning(coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689), 37), ENSURE_REQUIREMENTS, [plan_id=1157]
(56) Sort
Input [4]: [_extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
Arguments: [coalesce(_extract_internalRefUUID#1688, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1688) ASC NULLS FIRST, coalesce(_extract_internalRefUUID#1689, ) ASC NULLS FIRST, isnull(_extract_internalRefUUID#1689) ASC NULLS FIRST], false, 0
(57) SortMergeJoin
Left keys [4]: [coalesce(product#1127, ), isnull(product#1127), coalesce(plant#1128, ), isnull(plant#1128)]
Right keys [4]: [coalesce(_extract_internalRefUUID#1688, ), isnull(_extract_internalRefUUID#1688), coalesce(_extract_internalRefUUID#1689, ), isnull(_extract_internalRefUUID#1689)]
Join type: LeftOuter
Join condition: None
(58) Project
Output [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Input [6]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687, _extract_internalRefUUID#1688, _extract_internalRefUUID#1689]
(59) Exchange
Input [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: hashpartitioning(product#1127, 37), ENSURE_REQUIREMENTS, [plan_id=1164]
(60) Sort
Input [4]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687]
Arguments: [product#1127 ASC NULLS FIRST], false, 0
(61) Filter
Input [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
Condition : isnotnull(internalUUID#1194)
(62) Project
Output [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Input [4]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200, systemId#1193]
(63) Exchange
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=1141]
(64) Exchange
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: hashpartitioning(internalUUID#1194, 37), ENSURE_REQUIREMENTS, [plan_id=1165]
(65) Sort
Input [3]: [internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
Arguments: [internalUUID#1194 ASC NULLS FIRST], false, 0
(66) SortMergeJoin
Left keys [1]: [product#1127]
Right keys [1]: [internalUUID#1194]
Join type: LeftOuter
Join condition: None
(67) Project
Output [3]: [product#1127, plant#1128, filter(unitOfMeasureConversions#1200, lambdafunction((((lambda u#1206.measurementUnit1.code = _extract_code#1686) AND (lambda u#1206.measurementUnit1.internalRefUUID = _extract_internalRefUUID#1687)) AND ((lambda u#1206.measurementUnit2.code = baseUnitOfMeasure#1198.code) AND (lambda u#1206.measurementUnit2.internalRefUUID = baseUnitOfMeasure#1198.internalRefUUID))), lambda u#1206, false)) AS filteredUnitOfMeasureConversions#1205]
Input [7]: [product#1127, plant#1128, _extract_code#1686, _extract_internalRefUUID#1687, internalUUID#1194, baseUnitOfMeasure#1198, unitOfMeasureConversions#1200]
(68) Generate
Input [3]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205]
Arguments: explode(filteredUnitOfMeasureConversions#1205.quantityNumerator), [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205], true, [quantityNumerator#1213]
(69) Project
Output [3]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205.quantityDenominator AS _extract_quantityDenominator#1607]
Input [4]: [product#1127, plant#1128, filteredUnitOfMeasureConversions#1205, quantityNumerator#1213]
(70) Generate
Input [3]: [product#1127, plant#1128, _extract_quantityDenominator#1607]
Arguments: explode(_extract_quantityDenominator#1607), [product#1127, plant#1128], true, [quantityDenominator#1216]
(71) Project
Output [2]: [product#1127, plant#1128]
Input [3]: [product#1127, plant#1128, quantityDenominator#1216]
(72) BroadcastExchange
Input [4]: [product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Arguments: HashedRelationBroadcastMode(List(coalesce(input[1, string, true], ), isnull(input[1, string, true]), coalesce(input[0, string, true], ), isnull(input[0, string, true])),false), [plan_id=1175]
(73) BroadcastHashJoin
Left keys [4]: [coalesce(plant#1128, ), isnull(plant#1128), coalesce(product#1127, ), isnull(product#1127)]
Right keys [4]: [coalesce(plant#1573, ), isnull(plant#1573), coalesce(product#1572, ), isnull(product#1572)]
Join type: LeftOuter
Join condition: None
(74) Filter
Input [6]: [product#1127, plant#1128, product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
Condition : (((size(demandTimeBuckets#1577, true) = 0) OR isnull(demandTimeBuckets#1577)) AND ((size(demandPointInTime#1580, true) = 0) OR isnull(demandPointInTime#1580)))
(75) Project
Output [2]: [product#1127, plant#1128]
Input [6]: [product#1127, plant#1128, product#1572, plant#1573, demandTimeBuckets#1577, demandPointInTime#1580]
(76) HashAggregate
Input [2]: [product#1127, plant#1128]
Keys [2]: [product#1127, plant#1128]
Functions: []
Aggregate Attributes: []
Results [2]: [product#1127, plant#1128]
(77) HashAggregate
Input [2]: [product#1127, plant#1128]
Keys [2]: [product#1127, plant#1128]
Functions: []
Aggregate Attributes: []
Results [2]: [product#1127, plant#1128]
(78) AdaptiveSparkPlan
Output [2]: [product#1127, plant#1128]
Arguments: isFinalPlan=true
(79) TableCacheQueryStage
Output [2]: [product#1127, plant#1128]
Arguments: 0
(80) Filter [codegen id : 2]
Input [2]: [product#1127, plant#1128]
Condition : (isnotnull(product#1127) AND isnotnull(plant#1128))
(81) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(product#1127, plant#1128, 37), ENSURE_REQUIREMENTS, [plan_id=2218]
(82) ShuffleQueryStage
Output [2]: [product#1127, plant#1128]
Arguments: 2
(83) Sort [codegen id : 3]
Input [2]: [product#1127, plant#1128]
Arguments: [product#1127 ASC NULLS FIRST, plant#1128 ASC NULLS FIRST], false, 0
(84) Scan parquet
Output [3]: [plant#2315, product#2316, systemId#2313]
Batched: true
Location: PreparedDeltaFileIndex [hdlfs://7da5304c-1f57-47fa-b75a-0f0b4349c280.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant]
PushedFilters: [IsNotNull(product.internalRefUUID), IsNotNull(plant.internalRefUUID)]
ReadSchema: struct<plant:struct<internalRefUUID:string>,product:struct<internalRefUUID:string>>
(85) ColumnarToRow [codegen id : 1]
Input [3]: [plant#2315, product#2316, systemId#2313]
(86) Filter [codegen id : 1]
Input [3]: [plant#2315, product#2316, systemId#2313]
Condition : (isnotnull(product#2316.internalRefUUID) AND isnotnull(plant#2315.internalRefUUID))
(87) Project [codegen id : 1]
Output [2]: [product#2316.internalRefUUID AS product#2322, plant#2315.internalRefUUID AS plant#2323]
Input [3]: [plant#2315, product#2316, systemId#2313]
(88) Exchange
Input [2]: [product#2322, plant#2323]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=2157]
(89) ShuffleQueryStage
Output [2]: [product#2322, plant#2323]
Arguments: 1
(90) Exchange
Input [2]: [product#2322, plant#2323]
Arguments: hashpartitioning(product#2322, plant#2323, 37), ENSURE_REQUIREMENTS, [plan_id=2167]
(91) ShuffleQueryStage
Output [2]: [product#2322, plant#2323]
Arguments: 3
(92) Sort [codegen id : 4]
Input [2]: [product#2322, plant#2323]
Arguments: [product#2322 ASC NULLS FIRST, plant#2323 ASC NULLS FIRST], false, 0
(93) SortMergeJoin [codegen id : 5]
Left keys [2]: [product#1127, plant#1128]
Right keys [2]: [product#2322, plant#2323]
Join type: Inner
Join condition: None
(94) Project [codegen id : 5]
Output: []
Input [4]: [product#1127, plant#1128, product#2322, plant#2323]
(95) HashAggregate [codegen id : 5]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#2545L]
Results [1]: [count#2546L]
(96) Exchange
Input [1]: [count#2546L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2349]
(97) ShuffleQueryStage
Output [1]: [count#2546L]
Arguments: 4
(98) HashAggregate [codegen id : 6]
Input [1]: [count#2546L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#2392L]
Results [1]: [count(1)#2392L AS count#2389L]
(99) ResultQueryStage
Output [1]: [count#2389L]
Arguments: 5
(100) Filter
Input [2]: [product#1127, plant#1128]
Condition : (isnotnull(product#1127) AND isnotnull(plant#1128))
(101) Exchange
Input [2]: [product#1127, plant#1128]
Arguments: hashpartitioning(product#1127, plant#1128, 37), ENSURE_REQUIREMENTS, [plan_id=2112]
(102) Sort
Input [2]: [product#1127, plant#1128]
Arguments: [product#1127 ASC NULLS FIRST, plant#1128 ASC NULLS FIRST], false, 0
(103) Filter
Input [3]: [plant#2315, product#2316, systemId#2313]
Condition : (isnotnull(product#2316.internalRefUUID) AND isnotnull(plant#2315.internalRefUUID))
(104) Project
Output [2]: [product#2316.internalRefUUID AS product#2322, plant#2315.internalRefUUID AS plant#2323]
Input [3]: [plant#2315, product#2316, systemId#2313]
(105) Exchange
Input [2]: [product#2322, plant#2323]
Arguments: RoundRobinPartitioning(37), REPARTITION_BY_NUM, [plan_id=2104]
(106) Exchange
Input [2]: [product#2322, plant#2323]
Arguments: hashpartitioning(product#2322, plant#2323, 37), ENSURE_REQUIREMENTS, [plan_id=2111]
(107) Sort
Input [2]: [product#2322, plant#2323]
Arguments: [product#2322 ASC NULLS FIRST, plant#2323 ASC NULLS FIRST], false, 0
(108) SortMergeJoin
Left keys [2]: [product#1127, plant#1128]
Right keys [2]: [product#2322, plant#2323]
Join type: Inner
Join condition: None
(109) Project
Output: []
Input [4]: [product#1127, plant#1128, product#2322, plant#2323]
(110) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#2545L]
Results [1]: [count#2546L]
(111) Exchange
Input [1]: [count#2546L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2119]
(112) HashAggregate
Input [1]: [count#2546L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#2392L]
Results [1]: [count(1)#2392L AS count#2389L]
(113) AdaptiveSparkPlan
Output [1]: [count#2389L]
Arguments: isFinalPlan=true