minor edits to run script
" path=data_path,\n",
" boto3_session=my_session\n",
" )\n",
+ " for i in range(4):\n",
" df = pd.concat([df, df])\n",
+ " \n",
+ " df['unique_id'] = range(len(df))\n",
" bucket = \"alpha-splink-db-testing\"\n",
" path = f\"s3://{bucket}/data/\"\n",
" df.drop([\"source_dataset\"], axis=1, inplace=True)\n",
" database=\"splink_awswrangler_test\",\n",
" table=table_name\n",
" );\n",
- " print(len(df))"
+ " print(len(df))\n",
+ " display(df.head(10))\n",
+ " del(df)"
]
},
{
" output_bucket=\"alpha-splink-db-testing\",\n",
" database_name=\"splink_awswrangler_test\"\n",
")\n",
- "\n",
"linker.train_u_using_random_sampling(target_rows=1e6)\n",
"\n",
"if small_link_job:\n",
" \"l.dob = r.dob and substr(l.postcode,1,2) = substr(r.postcode,1,2)\"\n",
" )\n",
"\n",
- "\n",
+ "p = time.time()\n",
"df = linker.predict()\n",
+ "print(f\"Predict step took {time.time()-p} seconds\")\n",
"\n",
"print(f\"Total time taken {time.time()-t} seconds\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "50a986c7-d081-4184-bb16-af70ba514095",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "654/60"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d6828270-aa3c-4238-ac7a-688015ebaad2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "353/60"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7b02a23e-3412-44c3-b19f-c2553771925c",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7295235f-9ea8-44bd-9c52-87df789a8db6",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
