# op= Optimus(master="local[*]",comm=True)
%load_ext autoreload
%autoreload 2
import sys
sys.path.append("..")
from optimus import Optimus
# Create optimus
op = Optimus(master="local", app_name= "optimus", comm=True)
..\optimus\helpers\functions.py:170: DeprecationWarning: invalid escape sequence \d pattern = '\"(\d+\.\d+).*\"' C:\Users\argenisleon\Anaconda3\lib\site-packages\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working from collections import Callable You are using PySparkling of version 2.4.10, but your PySpark is of version 2.3.1. Please make sure Spark and PySparkling versions are compatible. `formatargspec` is deprecated since Python 3.5. Use `signature` and the `Signature` object directly invalid escape sequence \d
df_small= op.load.csv("data/foo.csv")
df_small.send("small")
'stddev' function in 'product' column is returning 'nan'. Is that what you expected?. Seems that 'product' has 'nan' values 'kurtosis' function in 'product' column is returning 'nan'. Is that what you expected?. Seems that 'product' has 'nan' values 'skewness' function in 'product' column is returning 'nan'. Is that what you expected?. Seems that 'product' has 'nan' values 'variance' function in 'product' column is returning 'nan'. Is that what you expected?. Seems that 'product' has 'nan' values Send!
df = op.load.csv("data/crime.csv")
%%time
df.send("crime")
'stddev' function in 'INCIDENT_NUMBER' column is returning 'nan'. Is that what you expected?. Seems that 'INCIDENT_NUMBER' has 'nan' values 'kurtosis' function in 'INCIDENT_NUMBER' column is returning 'nan'. Is that what you expected?. Seems that 'INCIDENT_NUMBER' has 'nan' values 'skewness' function in 'INCIDENT_NUMBER' column is returning 'nan'. Is that what you expected?. Seems that 'INCIDENT_NUMBER' has 'nan' values 'variance' function in 'INCIDENT_NUMBER' column is returning 'nan'. Is that what you expected?. Seems that 'INCIDENT_NUMBER' has 'nan' values 'kurtosis' function in 'STREET' column is returning 'nan'. Is that what you expected?. Seems that 'STREET' has 'nan' values 'skewness' function in 'STREET' column is returning 'nan'. Is that what you expected?. Seems that 'STREET' has 'nan' values Send! Wall time: 2min 51s
Get data from mysql and send it to Bumblebee
%%time
# Put your db credentials here
db = op.connect(
db_type="mysql",
host="165.227.196.70",
database= "optimus",
user= "test",
password = "test")
Wall time: 0 ns
Check the tables in the database
db.tables()
TABLE_NAME
1 (string)
nullable
|
TABLE_ROWS
2 (decimal(20,0))
nullable
|
---|---|
test_data
|
100
|
df_db = db.table_to_df("test_data",limit="all")
100 rows
df_db.send("db")
Send!