casey
/
Polaryx
Archived
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added select tool, but still need to handle ordering

main
Casey 2024-08-11 21:01:36 +10:00
parent b7696e80ab
commit 2374fe0d15
2 changed files with 215 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
Alteryx_datatypes.ods Normal file
View File

Binary file not shown.

368
scratchpad.ipynb
View File

@ -9,12 +9,11 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 23, "execution_count": 293,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Open Alteryx XML into a string \n", "# Open Alteryx XML into a string \n",
"\n",
"import polars as pl \n", "import polars as pl \n",
"import xml.etree.ElementTree as ET\n", "import xml.etree.ElementTree as ET\n",
"\n", "\n",
@ -27,12 +26,11 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 24, "execution_count": 294,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Parse out nodes (tool data) into a dict\n", "# Parse out nodes (tool data) into a dict\n",
"\n",
"def extract_tool_id_and_contents(xml_string):\n", "def extract_tool_id_and_contents(xml_string):\n",
" root = ET.fromstring(xml_string)\n", " root = ET.fromstring(xml_string)\n",
" results = []\n", " results = []\n",
@ -55,27 +53,41 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 25, "execution_count": 254,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def selectTool(df: pl.DataFrame, col_specs: dict):\n", "def tool_select(col_spec: dict):\n",
" \"\"\"\n", " \"\"\" Generates select tool code\"\"\"\n",
" Reshape a Polars DataFrame by renaming and retyping columns according to the provided dictionary.\n", " dynamic_code = \"df_output = df.with_columns(\\n\"\n",
" dynamic_code_suffix = ''\n",
" for old_name, (new_name, type, selected) in col_spec.items():\n",
"\n", "\n",
" Args:\n", " if old_name == '*Unknown':\n",
" df (pl.DataFrame): The input Polars DataFrame.\n", " break \n",
" col_specs (dict): A dictionary where keys are column names in the original DataFrame,\n", " \n",
" and values are tuples containing the new column name and data type.\n", " if new_name:\n",
" alias = f\".alias('{new_name}')\"\n",
" dynamic_code_suffix += f\"df_output = df_output.drop(f'{old_name}')\\n\"\n",
" else:\n",
" alias = ''\n",
"\n", "\n",
" Returns:\n", " if type is not None:\n",
" pl.DataFrame: The reshaped Polars DataFrame with renamed and retyped columns.\n", " if 'Int' in type:\n",
" \"\"\"\n", " cast = f\".cast(pl.{pl.Int64})\"\n",
" for old_name, (new_name, dt) in col_specs.items():\n", " elif 'String' in type:\n",
" df = df.rename({old_name: new_name})\n", " cast = f\".cast(pl.{pl.String})\"\n",
" if dt is not None:\n", " else:\n",
" df = df.with_column(pl.col(old_name).cast(dt))\n", " cast = ''\n",
" return df" "\n",
" if selected != 'False':\n",
" dynamic_code += f\"df.select(pl.col(f'{old_name}'){cast}{alias}),\\n\"\n",
" else:\n",
" dynamic_code_suffix += f\"df_output = df_output.drop(f'{old_name}')\\n\"\n",
"\n",
" dynamic_code += \")\\n\" + dynamic_code_suffix\n",
" \n",
" return dynamic_code"
] ]
}, },
{ {
@ -87,26 +99,139 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 26, "execution_count": 295,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def TextInputToDf(xml_string):\n", "def input_textInput(xml_string):\n",
" # Get XML for a Text input tool\n", " # Get XML for a Text input tool\n",
" root = ET.fromstring(results[3][1])\n", " root = ET.fromstring(xml_string)\n",
" # Extract the field names\n", " # Extract the field names\n",
" fields = [field.attrib['name'] for field in root.findall(\".//Fields/Field\")]\n", " fields = [field.attrib['name'] for field in root.findall(\".//Fields/Field\")]\n",
" # Extract the data rows\n", " # Extract the data rows\n",
" data_rows = [[int(c.text) for c in row.findall(\"c\")] for row in root.findall(\".//Data/r\")]\n", " data_rows = [[int(c.text) for c in row.findall(\"c\")] for row in root.findall(\".//Data/r\")]\n",
" # Create the polars dataframe\n", " # Create the polars dataframe\n",
" df = pl.DataFrame(data_rows, fields)\n", " df = pl.DataFrame(data_rows, fields, orient=\"row\")\n",
" # Display the dataframe\n", " # Display the dataframe\n",
" return df" " return df"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 22, "execution_count": 296,
"metadata": {},
"outputs": [],
"source": [
"def getConf_Select(xml_string):\n",
" root = ET.fromstring(xml_string)\n",
" dict_SelectTool = {}\n",
"\n",
" for field in root.findall(\".//SelectFields/SelectField\"):\n",
" field_name = field.attrib['field']\n",
" field_selected = field.attrib['selected']\n",
"\n",
" try:\n",
" field_type = field.attrib['type']\n",
" except:\n",
" field_type = None\n",
" try:\n",
" field_rename = field.attrib['rename']\n",
" except:\n",
" field_rename = None\n",
"\n",
" dict_SelectTool[field_name] = (field_rename, field_type, field_selected)\n",
"\n",
" return dict_SelectTool"
]
},
{
"cell_type": "code",
"execution_count": 313,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<Node ToolID=\"7\">\n",
" <GuiSettings Plugin=\"AlteryxBasePluginsGui.Join.Join\">\n",
" <Position x=\"318\" y=\"258\" />\n",
" </GuiSettings>\n",
" <Properties>\n",
" <Configuration joinByRecordPos=\"False\">\n",
" <JoinInfo connection=\"Left\">\n",
" <Field field=\"Col_3_renamed\" />\n",
" </JoinInfo>\n",
" <JoinInfo connection=\"Right\">\n",
" <Field field=\"Column 3\" />\n",
" </JoinInfo>\n",
" <SelectConfiguration>\n",
" <Configuration outputConnection=\"Join\">\n",
" <OrderChanged value=\"False\" />\n",
" <CommaDecimal value=\"False\" />\n",
" <SelectFields>\n",
" <SelectField field=\"Right_Column 3\" selected=\"True\" rename=\"Right_Column 3\" input=\"Right_\" />\n",
" <SelectField field=\"Right_Column 4\" selected=\"True\" rename=\"Right_Column 4\" input=\"Right_\" />\n",
" <SelectField field=\"*Unknown\" selected=\"True\" />\n",
" </SelectFields>\n",
" </Configuration>\n",
" </SelectConfiguration>\n",
" </Configuration>\n",
" <Annotation DisplayMode=\"0\">\n",
" <Name />\n",
" <DefaultAnnotationText />\n",
" <Left value=\"False\" />\n",
" </Annotation>\n",
" </Properties>\n",
" <EngineSettings EngineDll=\"AlteryxBasePluginsEngine.dll\" EngineDllEntryPoint=\"AlteryxJoin\" />\n",
" </Node>\n",
" \n",
"joinByRecordPos: False\n",
"{'joinParams': ('Col_3_renamed', 'Column 3')}\n"
]
}
],
"source": [
"### WIP\n",
"\n",
"def getConf_Join(xml_string):\n",
" print(xml_string)\n",
" root = ET.fromstring(xml_string)\n",
" dict_JoinTool = {}\n",
"\n",
" print(\"joinByRecordPos:\", root.find(\".//Properties/Configuration\").attrib['joinByRecordPos'])\n",
"\n",
" # Join parameters\n",
" for joinField in root.findall(\".//Configuration/JoinInfo\"):\n",
" if joinField.attrib['connection'] == \"Left\":\n",
" left_on = joinField.find('Field').attrib['field']\n",
" elif joinField.attrib['connection'] == \"Right\":\n",
" right_on = joinField.find('Field').attrib['field']\n",
"\n",
" if left_on == right_on:\n",
" dict_JoinTool['joinParams'] = ('on', left_on)\n",
" else:\n",
" dict_JoinTool['joinParams'] = (left_on, right_on)\n",
"\n",
" # Select parameters\n",
" for joinField in root.findall(\".//Configuration/SelectConfiguration\"):\n",
"\n",
" return dict_JoinTool\n",
"\n",
"xml_join_tool = results[6][1]\n",
"print(getConf_Join(xml_join_tool))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Working with the XML file"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
@ -151,7 +276,7 @@
"name": "stderr", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"C:\\Users\\casey.morter\\AppData\\Local\\Temp\\ipykernel_4012\\3571569777.py:9: DataOrientationWarning: Row orientation inferred during DataFrame construction. Explicitly specify the orientation by passing `orient=\"row\"` to silence this warning.\n", "/tmp/ipykernel_579015/219306832.py:9: DataOrientationWarning: Row orientation inferred during DataFrame construction. Explicitly specify the orientation by passing `orient=\"row\"` to silence this warning.\n",
" df = pl.DataFrame(data_rows, fields)\n" " df = pl.DataFrame(data_rows, fields)\n"
] ]
} }
@ -167,163 +292,100 @@
" print(ToolID, ToolType)\n", " print(ToolID, ToolType)\n",
"\n", "\n",
" if ToolType == 'TextInput':\n", " if ToolType == 'TextInput':\n",
" print(TextInputToDf(ToolXML))" " print(input_textInput(ToolXML))"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 29, "execution_count": 292,
"metadata": {},
"outputs": [],
"source": [
"# Check out a tool\n",
"# 0 = TextInput\n",
"# 4 = select with rename\n",
"tool_xml = results[4][1]"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"data": { "name": "stdout",
"text/plain": [ "output_type": "stream",
"'c:\\\\Users\\\\casey.morter\\\\OneDrive - JLL\\\\Documents\\\\01 Workspace\\\\01 Python\\\\Polaryx'" "text": [
] "\n",
}, "############### Input dataframe (TextInput):\n",
"execution_count": 27, "\n",
"metadata": {}, " shape: (3, 2)\n",
"output_type": "execute_result" "┌──────────┬──────────┐\n",
} "│ Column 3 ┆ Column 4 │\n",
], "│ --- ┆ --- │\n",
"source": [ "│ i64 ┆ i64 │\n",
"%pwd" "╞══════════╪══════════╡\n",
] "│ 5 ┆ 8 │\n",
}, "│ 6 ┆ 9 │\n",
{ "│ 7 ┆ 10 │\n",
"cell_type": "code", "└──────────┴──────────┘\n",
"execution_count": 31, "\n",
"metadata": {}, "############### Generated code from Select tool: \n",
"outputs": [ "\n",
" df_output = df.with_columns(\n",
"df.select(pl.col(f'Column 3').alias('Col_3_renamed')),\n",
")\n",
"df_output = df_output.drop(f'Column 3')\n",
"df_output = df_output.drop(f'Column 4')\n",
"\n",
"shape: (3, 1)\n",
"┌───────────────┐\n",
"│ Col_3_renamed │\n",
"│ --- │\n",
"│ i64 │\n",
"╞═══════════════╡\n",
"│ 5 │\n",
"│ 6 │\n",
"│ 7 │\n",
"└───────────────┘\n"
]
},
{ {
"ename": "KeyError", "name": "stderr",
"evalue": "'type'", "output_type": "stream",
"output_type": "error", "text": [
"traceback": [ "/tmp/ipykernel_579015/3571569777.py:9: DataOrientationWarning: Row orientation inferred during DataFrame construction. Explicitly specify the orientation by passing `orient=\"row\"` to silence this warning.\n",
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", " df = pl.DataFrame(data_rows, fields)\n"
"\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[1;32mIn[31], line 16\u001b[0m\n\u001b[0;32m 14\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m field \u001b[38;5;129;01min\u001b[39;00m root\u001b[38;5;241m.\u001b[39mfindall(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.//SelectFields/SelectField\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m 15\u001b[0m field_name \u001b[38;5;241m=\u001b[39m field\u001b[38;5;241m.\u001b[39mattrib[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfield\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m---> 16\u001b[0m field_type \u001b[38;5;241m=\u001b[39m \u001b[43mfield\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattrib\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtype\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m 19\u001b[0m field_rename \u001b[38;5;241m=\u001b[39m field\u001b[38;5;241m.\u001b[39mattrib[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrename\u001b[39m\u001b[38;5;124m'\u001b[39m]\n",
"\u001b[1;31mKeyError\u001b[0m: 'type'"
] ]
} }
], ],
"source": [ "source": [
"col_specs = {\n", "# Tool 1: TextInput\n",
" \"A\": (\"x\", int, False),\n", "xml_tool1 = results[3][1]\n",
" \"B\": (\"y\", str, False),\n", "df_in = TextInputToDf(ToolXML)\n",
" \"D\": (None, None, True) # drop this column\n", "\n",
"}\n", "print(\"\\n############### Input dataframe (TextInput):\\n\\n\", df_in)\n",
"\n",
"# Tool 2: Select\n",
"col_spec = getConf_Select(results[4][1])\n",
"\n",
"# print(\"\\n############### Select tool spec taken from XML:\\n\\n\", col_spec)\n",
"\n", "\n",
"\n", "\n",
"# Generate code from \n",
"code = tool_select(col_spec)\n",
"\n", "\n",
"# Parse the XML\n", "print(\"\\n############### Generated code from Select tool: \\n\\n\", code)\n",
"root = ET.fromstring(tool_xml)\n",
"\n", "\n",
"dict_SelectTool = {}\n", "df = df_in\n",
"\n", "\n",
"for field in root.findall(\".//SelectFields/SelectField\"):\n", "exec(code)\n",
" field_name = field.attrib['field']\n",
"\n", "\n",
" try:\n", "print(df_output)\n",
" field_type = field.attrib['type']\n", "\n"
" except:\n",
" field_type = None\n",
" \n",
" try:\n",
" field_rename = field.attrib['rename']\n",
" except:\n",
" field_rename = None\n",
"\n",
"\n",
"\n",
"\n",
" dict_SelectTool[field_name] = (None, None, field_rename)\n",
" \n",
"\n",
" # dict_SelectTool['field']\n",
"\n",
" # if field.attrib['field'] != '*Unknown':\n",
" # field.attrib['selected'], \"Type: \", field.attrib['size']\n",
"\n",
"\n",
"\n",
" \n",
" # print(field.attrib['field'], \"| Selected:\", field.attrib['selected'], \"Type: \", field.attrib['size'])\n",
"\n",
"dict_SelectTool"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 108, "execution_count": 174,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": []
"gui_settings_text = root.find(\".//GuiSettings\")"
]
},
{
"cell_type": "code",
"execution_count": 114,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'AlteryxBasePluginsGui.TextInput.TextInput'"
]
},
"execution_count": 114,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"gui_settings_text.attrib['Plugin']"
]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": []
"import polars as pl\n",
"\n",
"def reshape_polars_df(df: pl.DataFrame, col_specs: dict):\n",
" \"\"\"\n",
" Reshape a Polars DataFrame by renaming and retyping columns according to the provided dictionary.\n",
"\n",
" Args:\n",
" df (pl.DataFrame): The input Polars DataFrame.\n",
" col_specs (dict): A dictionary where keys are column names in the original DataFrame,\n",
" and values are tuples containing the new column name, data type, and a boolean indicating whether\n",
" the column should be dropped or not.\n",
"\n",
" Returns:\n",
" pl.DataFrame: The reshaped Polars DataFrame with renamed and retyped columns.\n",
" \"\"\"\n",
" for old_name, (new_name, dt, drop) in col_specs.items():\n",
" if drop:\n",
" df = df.drop(old_name)\n",
" else:\n",
" df = df.rename({old_name: new_name})\n",
" if dt is not None:\n",
" df = df.with_column(pl.col(old_name).cast(dt))\n",
" return df\n"
]
} }
], ],
"metadata": { "metadata": {