In [1]:
import numpy as np
import h5py
#f = h5py.File("/oak/stanford/groups/akundaje/avsec/basepair/data/"
# +"processed/comparison/output/nexus,peaks,OSNK,0,10"
# +",1,FALSE,same,0.5,64,25,0.004,9,FALSE,[1,50],TRUE"
# +",FALSE,1/deeplift.imp_score.h5", "r")
f = h5py.File("deeplift.imp_score.h5","r")
nanog_mask = np.array(f['metadata']['interval_from_task'][:]=='Nanog')
nanog_profile_wn_hypimp = np.array(f["hyp_imp/Nanog/profile/wn"][:])[nanog_mask]
onehot_seq = np.array(f["inputs/seq"][:])[nanog_mask]
nanog_profile_wn_contribs = nanog_profile_wn_hypimp*onehot_seq
In [2]:
import modisco
#track_set = modisco.tfmodisco_workflow.workflow.prep_track_set(
# task_names=["Nanog_profile_wn"],
# contrib_scores={'Nanog_profile_wn': nanog_profile_wn_contribs},
# hypothetical_contribs={'Nanog_profile_wn': nanog_profile_wn_hypimp},
# one_hot=onehot_seq)
#grp = h5py.File("/oak/stanford/groups/akundaje/avsec/basepair/data/processed/comparison/output/nexus"
# +",peaks,OSNK,0,10,1,FALSE,same,0.5,64,25,0.004,9,FALSE,[1,50],TRUE,FALSE,1/deeplift"
# +"/Nanog/out/profile/wn/modisco.h5","r")
#grp = h5py.File("modisco.h5","r")
#loaded_tfmodisco_results =\
# modisco.tfmodisco_workflow.workflow.TfModiscoResults.from_hdf5(grp, track_set=track_set)
#grp.close()
#patterns = (loaded_tfmodisco_results
# .metacluster_idx_to_submetacluster_results["metacluster_0"]
# .seqlets_to_patterns_result.patterns)
In [3]:
#Saving the seqlets
"""extracted_contrib_scores = []
extracted_hypothetical_scores = []
extracted_onehot_seqs = []
seqlets_list = loaded_tfmodisco_results.multitask_seqlet_creation_results.final_seqlets
window_around = 50
#extract +/- 50bp around each seqlet
for seqlet in seqlets_list:
example_idx = seqlet.coor.example_idx
start = seqlet.coor.start
end = seqlet.coor.end
if ((start>=window_around) and (end<=1000-window_around)):
extracted_contrib_scores.append(
nanog_profile_wn_contribs[example_idx,start-window_around:end+window_around])
extracted_hypothetical_scores.append(
nanog_profile_wn_hypimp[example_idx,start-window_around:end+window_around])
extracted_onehot_seqs.append(
onehot_seq[example_idx,start-window_around:end+window_around])
np.save("extracted_contrib_scores.npy", np.array(extracted_contrib_scores))
np.save("extracted_hypothetical_scores.npy", np.array(extracted_hypothetical_scores))
np.save("extracted_onehot.npy", np.array(extracted_onehot_seqs))"""
Out[3]:
'extracted_contrib_scores = []\nextracted_hypothetical_scores = []\nextracted_onehot_seqs = []\nseqlets_list = loaded_tfmodisco_results.multitask_seqlet_creation_results.final_seqlets\n\nwindow_around = 50\n\n#extract +/- 50bp around each seqlet\nfor seqlet in seqlets_list:\n example_idx = seqlet.coor.example_idx\n start = seqlet.coor.start\n end = seqlet.coor.end\n if ((start>=window_around) and (end<=1000-window_around)):\n extracted_contrib_scores.append(\n nanog_profile_wn_contribs[example_idx,start-window_around:end+window_around])\n extracted_hypothetical_scores.append(\n nanog_profile_wn_hypimp[example_idx,start-window_around:end+window_around])\n extracted_onehot_seqs.append(\n onehot_seq[example_idx,start-window_around:end+window_around])\n\nnp.save("extracted_contrib_scores.npy", np.array(extracted_contrib_scores))\nnp.save("extracted_hypothetical_scores.npy", np.array(extracted_hypothetical_scores))\nnp.save("extracted_onehot.npy", np.array(extracted_onehot_seqs))'
In [4]:
#for i in range(10):
# seqlets_list = loaded_tfmodisco_results.multitask_seqlet_creation_results.final_seqlets
# modisco.visualization.viz_sequence.plot_weights(seqlets_list[i]["Nanog_profile_wn_contrib_scores"].fwd)
In [5]:
#visualize the saved patterns:
"""%matplotlib inline
from modisco.visualization import viz_sequence
for idx,pattern in enumerate(patterns):
print("pattern idx",idx)
print(len(pattern.seqlets))
viz_sequence.plot_weights(
pattern["Nanog_profile_wn_contrib_scores"].fwd)
viz_sequence.plot_weights(pattern["sequence"].fwd)"""
Out[5]:
'%matplotlib inline\nfrom modisco.visualization import viz_sequence\nfor idx,pattern in enumerate(patterns):\n print("pattern idx",idx)\n print(len(pattern.seqlets))\n viz_sequence.plot_weights(\n pattern["Nanog_profile_wn_contrib_scores"].fwd)\n viz_sequence.plot_weights(pattern["sequence"].fwd)'
In [6]:
#print modisco commit hash
%cd /users/avanti/tfmodisco
!git log -n 1
%cd /users/avanti/tfmodisco_bio_experiments/bpnet/trial1
from importlib import reload
%matplotlib inline
import h5py
import numpy as np
import modisco
import modisco.seqlet_embedding.advanced_gapped_kmer
reload(modisco.seqlet_embedding.advanced_gapped_kmer)
import modisco.seqlet_embedding
reload(modisco.seqlet_embedding)
import modisco
reload(modisco)
reload(modisco.util)
import modisco.cluster.phenograph.core
reload(modisco.cluster.phenograph.core)
import modisco.cluster.phenograph.cluster
reload(modisco.cluster.phenograph.cluster)
import modisco.cluster.phenograph
reload(modisco.cluster.phenograph)
import modisco.cluster.core
reload(modisco.cluster.core)
import modisco.cluster
reload(modisco.cluster)
import modisco.affinitymat.core
reload(modisco.affinitymat.core)
import modisco.affinitymat.transformers
reload(modisco.affinitymat.transformers)
import modisco.tfmodisco_workflow.seqlets_to_patterns
reload(modisco.tfmodisco_workflow.seqlets_to_patterns)
import modisco.tfmodisco_workflow.workflow
reload(modisco.tfmodisco_workflow.workflow)
import modisco.nearest_neighbors
reload(modisco.nearest_neighbors)
import modisco.affinitymat
reload(modisco.affinitymat)
import modisco.aggregator
reload(modisco.aggregator)
import modisco.value_provider
reload(modisco.value_provider)
import modisco.core
reload(modisco.core)
import modisco.coordproducers
reload(modisco.coordproducers)
import modisco.metaclusterers
reload(modisco.metaclusterers)
import modisco.clusterinit.memeinit
reload(modisco.clusterinit.memeinit)
%matplotlib inline
N_CORES = 10
workflow = modisco.tfmodisco_workflow.workflow.TfModiscoWorkflow(
sliding_window_size=21,#[5,9,13,17,21],
flank_size=10,
target_seqlet_fdr=0.01,
min_passing_windows_frac=0.03,
max_passing_windows_frac=0.03,
min_metacluster_size=2000,
min_metacluster_size_frac=0.02,
max_seqlets_per_metacluster=50000,
seqlets_to_patterns_factory=
modisco.tfmodisco_workflow.seqlets_to_patterns.TfModiscoSeqletsToPatternsFactory(
#initclusterer_factory=modisco.clusterinit.memeinit.MemeInitClustererFactory(
# meme_command="/software/meme/5.0.1/bin/meme",
# base_outdir="meme_out",
# num_seqlets_to_use=10000,
# nmotifs=20, n_jobs=4),
use_louvain=False,
trim_to_window_size=30,
initial_flank_to_add=10,
embedder_factory=modisco.seqlet_embedding
.advanced_gapped_kmer
.AdvancedGappedKmerEmbedderFactory(n_jobs=N_CORES),
#kmer_len=6,
#num_gaps=2,
#num_mismatches=0,
n_cores=N_CORES,
final_min_cluster_size=60
)
)
results = workflow(
task_names=["Nanog_profile_wn"],
contrib_scores={'Nanog_profile_wn': nanog_profile_wn_contribs},
hypothetical_contribs={'Nanog_profile_wn': nanog_profile_wn_hypimp},
one_hot=onehot_seq)
/mnt/lab_data2/avanti/tfmodisco commit 94aaa2592830bc1a5aada6881bd381923c9859eb (HEAD -> parallel_leiden, origin/parallel_leiden) Author: Av Shrikumar <avanti.shrikumar@gmail.com> Date: Thu Feb 18 19:33:36 2021 -0800 debugged and tested the parallelization /mnt/lab_data2/avanti/tfmodisco_bio_experiments/bpnet/trial1 MEMORY 2.829688832 On task Nanog_profile_wn Computing windowed sums on original Generating null dist peak(mu)= 0.00754788601747714 Computing threshold Subsampling! For increasing = True , the minimum IR precision was 0.37577224214168486 occurring at 0.0 implying a frac_neg of 0.6019793855866568 To be conservative, adjusted frac neg is 0.95 For increasing = False , the minimum IR precision was 0.48571764379950555 occurring at -1.4484976418316364e-07 implying a frac_neg of 0.9444571409915278 To be conservative, adjusted frac neg is 0.95 Thresholds from null dist were -0.026598883792757988 and 0.25093573331832886 with frac passing 0.047041 Passing windows frac was 0.047041 , which is above 0.03 ; adjusting New thresholds are 0.31916173219680793 and -0.31916173219680793 Final raw thresholds are -0.31916173219680793 and 0.31916173219680793 Final transformed thresholds are -0.9701270905407408 and 0.9701270905407408
Got 97968 coords
After resolving overlaps, got 97968 seqlets
Across all tasks, the weakest transformed threshold used was: 0.9700270905407408
MEMORY 4.835258368
97968 identified in total
1 activity patterns with support >= 2000 out of 2 possible patterns
Metacluster sizes: [97965]
Idx to activities: {0: '1'}
MEMORY 4.835868672
On metacluster 0
Metacluster size 97965 limited to 50000
Relevant tasks: ('Nanog_profile_wn',)
Relevant signs: (1,)
TfModiscoSeqletsToPatternsFactory: seed=1234
(Round 1) num seqlets: 50000
(Round 1) Computing coarse affmat
MEMORY 4.835872768
Beginning embedding computation
MEMORY 4.835872768
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.0s [Parallel(n_jobs=10)]: Done 290 tasks | elapsed: 4.6s [Parallel(n_jobs=10)]: Done 790 tasks | elapsed: 9.2s [Parallel(n_jobs=10)]: Done 1490 tasks | elapsed: 15.3s [Parallel(n_jobs=10)]: Done 2390 tasks | elapsed: 23.1s [Parallel(n_jobs=10)]: Done 3490 tasks | elapsed: 32.6s [Parallel(n_jobs=10)]: Done 4790 tasks | elapsed: 43.6s [Parallel(n_jobs=10)]: Done 6290 tasks | elapsed: 57.0s [Parallel(n_jobs=10)]: Done 7990 tasks | elapsed: 1.2min [Parallel(n_jobs=10)]: Done 9890 tasks | elapsed: 1.5min [Parallel(n_jobs=10)]: Done 11990 tasks | elapsed: 1.8min [Parallel(n_jobs=10)]: Done 14290 tasks | elapsed: 2.1min [Parallel(n_jobs=10)]: Done 16790 tasks | elapsed: 2.4min [Parallel(n_jobs=10)]: Done 19490 tasks | elapsed: 2.8min [Parallel(n_jobs=10)]: Done 22390 tasks | elapsed: 3.2min [Parallel(n_jobs=10)]: Done 25490 tasks | elapsed: 3.7min [Parallel(n_jobs=10)]: Done 28750 tasks | elapsed: 4.1min [Parallel(n_jobs=10)]: Done 32220 tasks | elapsed: 4.7min [Parallel(n_jobs=10)]: Done 35920 tasks | elapsed: 5.2min [Parallel(n_jobs=10)]: Done 39780 tasks | elapsed: 5.7min [Parallel(n_jobs=10)]: Done 43840 tasks | elapsed: 6.4min [Parallel(n_jobs=10)]: Done 48100 tasks | elapsed: 7.0min [Parallel(n_jobs=10)]: Done 50000 out of 50000 | elapsed: 7.4min finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.5s [Parallel(n_jobs=10)]: Done 340 tasks | elapsed: 3.3s [Parallel(n_jobs=10)]: Done 840 tasks | elapsed: 7.8s [Parallel(n_jobs=10)]: Done 1540 tasks | elapsed: 14.0s [Parallel(n_jobs=10)]: Done 2400 tasks | elapsed: 25.1s [Parallel(n_jobs=10)]: Done 3500 tasks | elapsed: 34.6s [Parallel(n_jobs=10)]: Done 4800 tasks | elapsed: 45.6s [Parallel(n_jobs=10)]: Done 6300 tasks | elapsed: 58.1s [Parallel(n_jobs=10)]: Done 7960 tasks | elapsed: 1.3min [Parallel(n_jobs=10)]: Done 9860 tasks | elapsed: 1.5min [Parallel(n_jobs=10)]: Done 11960 tasks | elapsed: 1.8min [Parallel(n_jobs=10)]: Done 14220 tasks | elapsed: 2.2min [Parallel(n_jobs=10)]: Done 16720 tasks | elapsed: 2.6min [Parallel(n_jobs=10)]: Done 19380 tasks | elapsed: 3.0min [Parallel(n_jobs=10)]: Done 22280 tasks | elapsed: 3.4min [Parallel(n_jobs=10)]: Done 27020 tasks | elapsed: 4.1min [Parallel(n_jobs=10)]: Done 33210 tasks | elapsed: 5.1min [Parallel(n_jobs=10)]: Done 36710 tasks | elapsed: 5.6min [Parallel(n_jobs=10)]: Done 40380 tasks | elapsed: 6.2min [Parallel(n_jobs=10)]: Done 44240 tasks | elapsed: 6.9min [Parallel(n_jobs=10)]: Done 48340 tasks | elapsed: 7.5min [Parallel(n_jobs=10)]: Done 50000 out of 50000 | elapsed: 7.9min finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 50000 out of 50000 | elapsed: 3.1min finished
Constructing csr matrix... csr matrix made in 14.266992568969727 s
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 50000 out of 50000 | elapsed: 2.9min finished
Constructing csr matrix... csr matrix made in 15.30729627609253 s Finished embedding computation in 1369.97 s MEMORY 12.653821952 Starting affinity matrix computations MEMORY 12.653821952 Batching in slices of size 1342
100%|██████████| 38/38 [13:07<00:00, 20.72s/it]
Finished affinity matrix computations in 788.1 s MEMORY 12.653912064
(Round 1) Computed coarse affmat MEMORY 12.0528896 (Round 1) Computing affinity matrix on nearest neighbors MEMORY 12.0528896 Launching nearest neighbors affmat calculation job MEMORY 12.0528896 Parallel runs completed MEMORY 12.500967424 Job completed in: 370.45 s MEMORY 12.50070528 Launching nearest neighbors affmat calculation job MEMORY 12.50070528 Parallel runs completed MEMORY 12.929912832 Job completed in: 368.8 s MEMORY 12.929912832 (Round 1) Computed affinity matrix on nearest neighbors in 753.49 s MEMORY 12.913586176 Filtered down to 41997 of 50000 (Round 1) Retained 41997 rows out of 50000 after filtering MEMORY 12.913852416 (Round 1) Computing density adapted affmat MEMORY 12.913852416 [t-SNE] Computed conditional probabilities for sample 1000 / 41997 [t-SNE] Computed conditional probabilities for sample 2000 / 41997 [t-SNE] Computed conditional probabilities for sample 3000 / 41997 [t-SNE] Computed conditional probabilities for sample 4000 / 41997 [t-SNE] Computed conditional probabilities for sample 5000 / 41997 [t-SNE] Computed conditional probabilities for sample 6000 / 41997 [t-SNE] Computed conditional probabilities for sample 7000 / 41997 [t-SNE] Computed conditional probabilities for sample 8000 / 41997 [t-SNE] Computed conditional probabilities for sample 9000 / 41997 [t-SNE] Computed conditional probabilities for sample 10000 / 41997 [t-SNE] Computed conditional probabilities for sample 11000 / 41997 [t-SNE] Computed conditional probabilities for sample 12000 / 41997 [t-SNE] Computed conditional probabilities for sample 13000 / 41997 [t-SNE] Computed conditional probabilities for sample 14000 / 41997 [t-SNE] Computed conditional probabilities for sample 15000 / 41997 [t-SNE] Computed conditional probabilities for sample 16000 / 41997 [t-SNE] Computed conditional probabilities for sample 17000 / 41997 [t-SNE] Computed conditional probabilities for sample 18000 / 41997 [t-SNE] Computed conditional probabilities for sample 19000 / 41997 [t-SNE] Computed conditional probabilities for sample 20000 / 41997 [t-SNE] Computed conditional probabilities for sample 21000 / 41997 [t-SNE] Computed conditional probabilities for sample 22000 / 41997 [t-SNE] Computed conditional probabilities for sample 23000 / 41997 [t-SNE] Computed conditional probabilities for sample 24000 / 41997 [t-SNE] Computed conditional probabilities for sample 25000 / 41997 [t-SNE] Computed conditional probabilities for sample 26000 / 41997 [t-SNE] Computed conditional probabilities for sample 27000 / 41997 [t-SNE] Computed conditional probabilities for sample 28000 / 41997 [t-SNE] Computed conditional probabilities for sample 29000 / 41997 [t-SNE] Computed conditional probabilities for sample 30000 / 41997 [t-SNE] Computed conditional probabilities for sample 31000 / 41997 [t-SNE] Computed conditional probabilities for sample 32000 / 41997 [t-SNE] Computed conditional probabilities for sample 33000 / 41997 [t-SNE] Computed conditional probabilities for sample 34000 / 41997 [t-SNE] Computed conditional probabilities for sample 35000 / 41997 [t-SNE] Computed conditional probabilities for sample 36000 / 41997 [t-SNE] Computed conditional probabilities for sample 37000 / 41997 [t-SNE] Computed conditional probabilities for sample 38000 / 41997 [t-SNE] Computed conditional probabilities for sample 39000 / 41997 [t-SNE] Computed conditional probabilities for sample 40000 / 41997 [t-SNE] Computed conditional probabilities for sample 41000 / 41997 [t-SNE] Computed conditional probabilities for sample 41997 / 41997 [t-SNE] Mean sigma: 0.207874 (Round 1) Computing clustering MEMORY 12.913856512 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 5.0min
Quality: 0.7561596221748645 Quality: 0.756257737241806 Quality: 0.7563831136214795 Quality: 0.7564157093885451 Quality: 0.7566731316389486
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 8.2min finished
Got 36 clusters after round 1
Counts:
{17: 919, 7: 2063, 10: 1803, 12: 1581, 3: 2693, 8: 1961, 4: 2527, 11: 1774, 13: 1546, 5: 2225, 16: 1084, 14: 1380, 9: 1944, 1: 3419, 0: 4668, 20: 657, 2: 3028, 15: 1165, 18: 766, 6: 2137, 19: 683, 21: 592, 31: 20, 30: 40, 23: 344, 35: 10, 28: 58, 27: 64, 24: 133, 32: 20, 25: 93, 29: 44, 22: 446, 26: 83, 34: 12, 33: 15}
MEMORY 12.948992
(Round 1) Aggregating seqlets in each cluster
MEMORY 12.948992
Aggregating for cluster 0 with 4668 seqlets
MEMORY 12.948992
Trimming eliminated 0 seqlets out of 4668
Skipped 1 seqlets
Removed 12 duplicate seqlets
Aggregating for cluster 1 with 3419 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 3419
Skipped 1 seqlets
Removed 7 duplicate seqlets
Aggregating for cluster 2 with 3028 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 3028
Removed 14 duplicate seqlets
Aggregating for cluster 3 with 2693 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 2693
Removed 27 duplicate seqlets
Aggregating for cluster 4 with 2527 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 2527
Removed 22 duplicate seqlets
Aggregating for cluster 5 with 2225 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 2225
Skipped 2 seqlets
Removed 7 duplicate seqlets
Aggregating for cluster 6 with 2137 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 2137
Aggregating for cluster 7 with 2063 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 2063
Skipped 3 seqlets
Removed 19 duplicate seqlets
Aggregating for cluster 8 with 1961 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1961
Skipped 1 seqlets
Removed 13 duplicate seqlets
Aggregating for cluster 9 with 1944 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1944
Removed 5 duplicate seqlets
Aggregating for cluster 10 with 1803 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1803
Skipped 1 seqlets
Removed 10 duplicate seqlets
Aggregating for cluster 11 with 1774 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1774
Removed 29 duplicate seqlets
Aggregating for cluster 12 with 1581 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1581
Removed 5 duplicate seqlets
Removed 1 duplicate seqlets
Aggregating for cluster 13 with 1546 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1546
Removed 5 duplicate seqlets
Aggregating for cluster 14 with 1380 seqlets
MEMORY 12.948217856
Trimming eliminated 0 seqlets out of 1380
Removed 4 duplicate seqlets
Aggregating for cluster 15 with 1165 seqlets
MEMORY 12.94821376
Trimming eliminated 0 seqlets out of 1165
Skipped 1 seqlets
Removed 3 duplicate seqlets
Aggregating for cluster 16 with 1084 seqlets
MEMORY 12.948205568
Trimming eliminated 0 seqlets out of 1084
Removed 4 duplicate seqlets
Aggregating for cluster 17 with 919 seqlets
MEMORY 12.948201472
Trimming eliminated 0 seqlets out of 919
Removed 3 duplicate seqlets
Aggregating for cluster 18 with 766 seqlets
MEMORY 12.948197376
Trimming eliminated 0 seqlets out of 766
Aggregating for cluster 19 with 683 seqlets
MEMORY 12.948197376
Trimming eliminated 0 seqlets out of 683
Removed 1 duplicate seqlets
Aggregating for cluster 20 with 657 seqlets
MEMORY 12.948197376
Trimming eliminated 0 seqlets out of 657
Removed 2 duplicate seqlets
Aggregating for cluster 21 with 592 seqlets
MEMORY 12.94819328
Trimming eliminated 0 seqlets out of 592
Removed 1 duplicate seqlets
Aggregating for cluster 22 with 446 seqlets
MEMORY 12.94819328
Trimming eliminated 0 seqlets out of 446
Aggregating for cluster 23 with 344 seqlets
MEMORY 12.94819328
Trimming eliminated 0 seqlets out of 344
Aggregating for cluster 24 with 133 seqlets
MEMORY 12.948189184
Trimming eliminated 0 seqlets out of 133
Aggregating for cluster 25 with 93 seqlets
MEMORY 12.948189184
Trimming eliminated 0 seqlets out of 93
Removed 2 duplicate seqlets
Aggregating for cluster 26 with 83 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 83
Aggregating for cluster 27 with 64 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 64
Removed 3 duplicate seqlets
Aggregating for cluster 28 with 58 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 58
Aggregating for cluster 29 with 44 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 44
Aggregating for cluster 30 with 40 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 40
Removed 4 duplicate seqlets
Aggregating for cluster 31 with 20 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 20
Aggregating for cluster 32 with 20 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 20
Aggregating for cluster 33 with 15 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 15
Aggregating for cluster 34 with 12 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 12
Aggregating for cluster 35 with 10 seqlets
MEMORY 12.948185088
Trimming eliminated 0 seqlets out of 10
(Round 2) num seqlets: 41729
(Round 2) Computing coarse affmat
MEMORY 12.948185088
Beginning embedding computation
MEMORY 12.948185088
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 31 tasks | elapsed: 3.2s [Parallel(n_jobs=10)]: Done 444 tasks | elapsed: 6.3s [Parallel(n_jobs=10)]: Done 1444 tasks | elapsed: 13.5s [Parallel(n_jobs=10)]: Done 2844 tasks | elapsed: 23.4s [Parallel(n_jobs=10)]: Done 4644 tasks | elapsed: 36.2s [Parallel(n_jobs=10)]: Done 6844 tasks | elapsed: 52.3s [Parallel(n_jobs=10)]: Done 9444 tasks | elapsed: 1.2min [Parallel(n_jobs=10)]: Done 12444 tasks | elapsed: 1.5min [Parallel(n_jobs=10)]: Done 15714 tasks | elapsed: 1.9min [Parallel(n_jobs=10)]: Done 19514 tasks | elapsed: 2.3min [Parallel(n_jobs=10)]: Done 23714 tasks | elapsed: 2.7min [Parallel(n_jobs=10)]: Done 28184 tasks | elapsed: 3.2min [Parallel(n_jobs=10)]: Done 33184 tasks | elapsed: 3.8min [Parallel(n_jobs=10)]: Done 38551 tasks | elapsed: 4.4min [Parallel(n_jobs=10)]: Done 41729 out of 41729 | elapsed: 4.7min finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.4s [Parallel(n_jobs=10)]: Done 620 tasks | elapsed: 4.5s [Parallel(n_jobs=10)]: Done 1620 tasks | elapsed: 11.7s [Parallel(n_jobs=10)]: Done 3020 tasks | elapsed: 21.6s [Parallel(n_jobs=10)]: Done 4820 tasks | elapsed: 34.6s [Parallel(n_jobs=10)]: Done 7020 tasks | elapsed: 50.2s [Parallel(n_jobs=10)]: Done 9620 tasks | elapsed: 1.1min [Parallel(n_jobs=10)]: Done 12460 tasks | elapsed: 1.5min [Parallel(n_jobs=10)]: Done 15860 tasks | elapsed: 1.9min [Parallel(n_jobs=10)]: Done 19660 tasks | elapsed: 2.2min [Parallel(n_jobs=10)]: Done 23700 tasks | elapsed: 2.8min [Parallel(n_jobs=10)]: Done 28300 tasks | elapsed: 3.2min [Parallel(n_jobs=10)]: Done 33300 tasks | elapsed: 3.8min [Parallel(n_jobs=10)]: Done 38570 tasks | elapsed: 4.4min [Parallel(n_jobs=10)]: Done 41729 out of 41729 | elapsed: 4.8min finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 41729 out of 41729 | elapsed: 2.7min finished
Constructing csr matrix... csr matrix made in 13.166661024093628 s
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 41729 out of 41729 | elapsed: 2.5min finished
Constructing csr matrix... csr matrix made in 13.20762848854065 s Finished embedding computation in 965.04 s MEMORY 14.19200512 Starting affinity matrix computations MEMORY 14.191742976 Batching in slices of size 1608
100%|██████████| 26/26 [08:52<00:00, 20.47s/it]
Finished affinity matrix computations in 532.84 s MEMORY 14.191742976
(Round 2) Computed coarse affmat MEMORY 13.693857792 (Round 2) Computing affinity matrix on nearest neighbors MEMORY 13.693857792 Launching nearest neighbors affmat calculation job MEMORY 13.93422336 Parallel runs completed MEMORY 13.949874176 Job completed in: 458.88 s MEMORY 13.949612032 Launching nearest neighbors affmat calculation job MEMORY 13.949612032 Parallel runs completed MEMORY 14.030102528 Job completed in: 507.94 s MEMORY 14.029840384 (Round 2) Computed affinity matrix on nearest neighbors in 979.18 s MEMORY 13.750153216 Not applying filtering for rounds above first round MEMORY 13.750153216 (Round 2) Computing density adapted affmat MEMORY 13.750153216 [t-SNE] Computed conditional probabilities for sample 1000 / 41729 [t-SNE] Computed conditional probabilities for sample 2000 / 41729 [t-SNE] Computed conditional probabilities for sample 3000 / 41729 [t-SNE] Computed conditional probabilities for sample 4000 / 41729 [t-SNE] Computed conditional probabilities for sample 5000 / 41729 [t-SNE] Computed conditional probabilities for sample 6000 / 41729 [t-SNE] Computed conditional probabilities for sample 7000 / 41729 [t-SNE] Computed conditional probabilities for sample 8000 / 41729 [t-SNE] Computed conditional probabilities for sample 9000 / 41729 [t-SNE] Computed conditional probabilities for sample 10000 / 41729 [t-SNE] Computed conditional probabilities for sample 11000 / 41729 [t-SNE] Computed conditional probabilities for sample 12000 / 41729 [t-SNE] Computed conditional probabilities for sample 13000 / 41729 [t-SNE] Computed conditional probabilities for sample 14000 / 41729 [t-SNE] Computed conditional probabilities for sample 15000 / 41729 [t-SNE] Computed conditional probabilities for sample 16000 / 41729 [t-SNE] Computed conditional probabilities for sample 17000 / 41729 [t-SNE] Computed conditional probabilities for sample 18000 / 41729 [t-SNE] Computed conditional probabilities for sample 19000 / 41729 [t-SNE] Computed conditional probabilities for sample 20000 / 41729 [t-SNE] Computed conditional probabilities for sample 21000 / 41729 [t-SNE] Computed conditional probabilities for sample 22000 / 41729 [t-SNE] Computed conditional probabilities for sample 23000 / 41729 [t-SNE] Computed conditional probabilities for sample 24000 / 41729 [t-SNE] Computed conditional probabilities for sample 25000 / 41729 [t-SNE] Computed conditional probabilities for sample 26000 / 41729 [t-SNE] Computed conditional probabilities for sample 27000 / 41729 [t-SNE] Computed conditional probabilities for sample 28000 / 41729 [t-SNE] Computed conditional probabilities for sample 29000 / 41729 [t-SNE] Computed conditional probabilities for sample 30000 / 41729 [t-SNE] Computed conditional probabilities for sample 31000 / 41729 [t-SNE] Computed conditional probabilities for sample 32000 / 41729 [t-SNE] Computed conditional probabilities for sample 33000 / 41729 [t-SNE] Computed conditional probabilities for sample 34000 / 41729 [t-SNE] Computed conditional probabilities for sample 35000 / 41729 [t-SNE] Computed conditional probabilities for sample 36000 / 41729 [t-SNE] Computed conditional probabilities for sample 37000 / 41729 [t-SNE] Computed conditional probabilities for sample 38000 / 41729 [t-SNE] Computed conditional probabilities for sample 39000 / 41729 [t-SNE] Computed conditional probabilities for sample 40000 / 41729 [t-SNE] Computed conditional probabilities for sample 41000 / 41729 [t-SNE] Computed conditional probabilities for sample 41729 / 41729 [t-SNE] Mean sigma: 0.219380 (Round 2) Computing clustering MEMORY 13.79475456 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 5.9min
Quality: 0.7322010420713774 Quality: 0.7340107999623549 Quality: 0.7342343444473574 Quality: 0.7342852098214336 Quality: 0.7348613971762101 Quality: 0.735031852765332 Quality: 0.7350370206481343
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 8.8min finished
Got 39 clusters after round 2
Counts:
{0: 4751, 3: 2771, 13: 1420, 1: 3509, 7: 2057, 11: 1767, 5: 2354, 12: 1686, 34: 20, 24: 256, 20: 524, 4: 2508, 14: 1185, 17: 820, 21: 496, 10: 1802, 8: 1994, 25: 177, 19: 542, 2: 3118, 6: 2065, 16: 874, 9: 1885, 30: 72, 36: 11, 15: 1028, 23: 263, 18: 758, 37: 10, 31: 46, 32: 44, 29: 76, 22: 438, 27: 119, 38: 10, 28: 81, 35: 16, 26: 156, 33: 20}
MEMORY 13.72727296
(Round 2) Aggregating seqlets in each cluster
MEMORY 13.72727296
Aggregating for cluster 0 with 4751 seqlets
MEMORY 13.72727296
Trimming eliminated 0 seqlets out of 4751
Removed 216 duplicate seqlets
Aggregating for cluster 1 with 3509 seqlets
MEMORY 13.726486528
Trimming eliminated 0 seqlets out of 3509
Skipped 1 seqlets
Removed 96 duplicate seqlets
Aggregating for cluster 2 with 3118 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 3118
Removed 74 duplicate seqlets
Aggregating for cluster 3 with 2771 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 2771
Removed 226 duplicate seqlets
Aggregating for cluster 4 with 2508 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 2508
Skipped 1 seqlets
Removed 170 duplicate seqlets
Aggregating for cluster 5 with 2354 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 2354
Removed 210 duplicate seqlets
Aggregating for cluster 6 with 2065 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 2065
Removed 84 duplicate seqlets
Aggregating for cluster 7 with 2057 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 2057
Skipped 1 seqlets
Removed 144 duplicate seqlets
Aggregating for cluster 8 with 1994 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 1994
Removed 132 duplicate seqlets
Aggregating for cluster 9 with 1885 seqlets
MEMORY 13.726322688
Trimming eliminated 0 seqlets out of 1885
Removed 73 duplicate seqlets
Aggregating for cluster 10 with 1802 seqlets
MEMORY 13.726314496
Trimming eliminated 0 seqlets out of 1802
Skipped 1 seqlets
Removed 88 duplicate seqlets
Aggregating for cluster 11 with 1767 seqlets
MEMORY 13.7263104
Trimming eliminated 0 seqlets out of 1767
Skipped 1 seqlets
Removed 105 duplicate seqlets
Aggregating for cluster 12 with 1686 seqlets
MEMORY 13.726306304
Trimming eliminated 0 seqlets out of 1686
Removed 288 duplicate seqlets
Aggregating for cluster 13 with 1420 seqlets
MEMORY 13.7263104
Trimming eliminated 0 seqlets out of 1420
Removed 92 duplicate seqlets
Aggregating for cluster 14 with 1185 seqlets
MEMORY 13.7263104
Trimming eliminated 0 seqlets out of 1185
Removed 93 duplicate seqlets
Aggregating for cluster 15 with 1028 seqlets
MEMORY 13.726302208
Trimming eliminated 0 seqlets out of 1028
Removed 51 duplicate seqlets
Aggregating for cluster 16 with 874 seqlets
MEMORY 13.726298112
Trimming eliminated 0 seqlets out of 874
Removed 53 duplicate seqlets
Aggregating for cluster 17 with 820 seqlets
MEMORY 13.72628992
Trimming eliminated 0 seqlets out of 820
Removed 38 duplicate seqlets
Aggregating for cluster 18 with 758 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 758
Removed 91 duplicate seqlets
Aggregating for cluster 19 with 542 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 542
Skipped 1 seqlets
Removed 33 duplicate seqlets
Aggregating for cluster 20 with 524 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 524
Removed 13 duplicate seqlets
Aggregating for cluster 21 with 496 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 496
Removed 24 duplicate seqlets
Aggregating for cluster 22 with 438 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 438
Removed 3 duplicate seqlets
Aggregating for cluster 23 with 263 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 263
Removed 15 duplicate seqlets
Aggregating for cluster 24 with 256 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 256
Removed 5 duplicate seqlets
Aggregating for cluster 25 with 177 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 177
Removed 7 duplicate seqlets
Aggregating for cluster 26 with 156 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 156
Removed 38 duplicate seqlets
Aggregating for cluster 27 with 119 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 119
Removed 1 duplicate seqlets
Aggregating for cluster 28 with 81 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 81
Removed 1 duplicate seqlets
Aggregating for cluster 29 with 76 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 76
Aggregating for cluster 30 with 72 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 72
Removed 1 duplicate seqlets
Aggregating for cluster 31 with 46 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 46
Aggregating for cluster 32 with 44 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 44
Aggregating for cluster 33 with 20 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 20
Removed 3 duplicate seqlets
Aggregating for cluster 34 with 20 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 20
Aggregating for cluster 35 with 16 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 16
Aggregating for cluster 36 with 11 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 11
Aggregating for cluster 37 with 10 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 10
Removed 1 duplicate seqlets
Aggregating for cluster 38 with 10 seqlets
MEMORY 13.726285824
Trimming eliminated 0 seqlets out of 10
Got 39 clusters
Splitting into subclusters...
MEMORY 13.726285824
Inspecting for spurious merging
Wrote graph to binary file in 34.8299834728241 seconds
MEMORY 14.733922304
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0035167
After 3 runs, maximum modularity is Q = 0.00353631
After 4 runs, maximum modularity is Q = 0.0036014
After 10 runs, maximum modularity is Q = 0.00360187
Louvain completed 30 runs in 94.92591190338135 seconds
Similarity is 0.93307614; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 20.66571354866028 seconds
MEMORY 14.331592704
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00602221
After 3 runs, maximum modularity is Q = 0.00602287
After 18 runs, maximum modularity is Q = 0.00602288
After 36 runs, maximum modularity is Q = 0.00602289
Louvain completed 56 runs in 94.45869421958923 seconds
Similarity is 0.8876958; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 16.390953063964844 seconds
MEMORY 14.20806144
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0079132
After 2 runs, maximum modularity is Q = 0.00821755
Louvain completed 22 runs in 31.630218267440796 seconds
Similarity is 0.9483532; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 11.832741975784302 seconds
MEMORY 14.069518336
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00570488
After 4 runs, maximum modularity is Q = 0.00570495
After 9 runs, maximum modularity is Q = 0.00570496
Louvain completed 29 runs in 39.60551118850708 seconds
Similarity is 0.82006204; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 10.939521789550781 seconds
MEMORY 14.015700992
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00549338
After 3 runs, maximum modularity is Q = 0.00549365
After 5 runs, maximum modularity is Q = 0.00549389
After 6 runs, maximum modularity is Q = 0.00549405
After 9 runs, maximum modularity is Q = 0.00549411
Louvain completed 29 runs in 36.823731899261475 seconds
Similarity is 0.8526772; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 8.206113815307617 seconds
MEMORY 13.965275136
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00529494
After 2 runs, maximum modularity is Q = 0.0052952
Louvain completed 22 runs in 25.859800338745117 seconds
Similarity is 0.84580207; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 7.087634325027466 seconds
MEMORY 13.930295296
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00454161
After 4 runs, maximum modularity is Q = 0.00454167
After 18 runs, maximum modularity is Q = 0.00454203
Louvain completed 38 runs in 47.29554462432861 seconds
Similarity is 0.8743118; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 6.5975542068481445 seconds
MEMORY 13.901832192
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0050971
After 2 runs, maximum modularity is Q = 0.00509741
After 3 runs, maximum modularity is Q = 0.00509749
Louvain completed 23 runs in 25.68368172645569 seconds
Similarity is 0.86179835; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 6.257678985595703 seconds
MEMORY 13.895798784
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00744901
Louvain completed 21 runs in 21.48015522956848 seconds
Similarity is 0.7859383; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 1.8310546875 seconds
MEMORY 13.812588544
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00531805
Louvain completed 21 runs in 17.11457395553589 seconds
Similarity is 0.86380917; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 1.2435457706451416 seconds
MEMORY 13.812588544
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00570864
After 2 runs, maximum modularity is Q = 0.00570877
Louvain completed 22 runs in 18.654019355773926 seconds
Similarity is 0.86664426; is_dissimilar is False
Got 2 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 5.865099668502808 seconds
MEMORY 13.883961344
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00697174
After 2 runs, maximum modularity is Q = 0.006972
After 3 runs, maximum modularity is Q = 0.00697203
Louvain completed 23 runs in 24.1513512134552 seconds
Similarity is 0.8594708; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 5.283619403839111 seconds
MEMORY 13.931352064
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00382016
After 2 runs, maximum modularity is Q = 0.00382029
Louvain completed 22 runs in 23.934102058410645 seconds
Similarity is 0.94444716; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 5.225455284118652 seconds
MEMORY 13.924872192
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00462296
After 2 runs, maximum modularity is Q = 0.00462366
After 9 runs, maximum modularity is Q = 0.00462373
After 17 runs, maximum modularity is Q = 0.00462376
Louvain completed 37 runs in 38.479421854019165 seconds
Similarity is 0.8680991; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 3.5269157886505127 seconds
MEMORY 13.87200512
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0114846
Louvain completed 21 runs in 20.504961013793945 seconds
Similarity is 0.90738046; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 3.2424325942993164 seconds
MEMORY 13.853212672
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0051117
After 2 runs, maximum modularity is Q = 0.00511176
After 6 runs, maximum modularity is Q = 0.00511185
After 8 runs, maximum modularity is Q = 0.00511186
After 11 runs, maximum modularity is Q = 0.00511198
After 31 runs, maximum modularity is Q = 0.00511209
Louvain completed 51 runs in 50.38367438316345 seconds
Similarity is 0.8621207; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 2.1519620418548584 seconds
MEMORY 13.792612352
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00578566
Louvain completed 21 runs in 17.278958082199097 seconds
Similarity is 0.83924794; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 1.838531494140625 seconds
MEMORY 13.792612352
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00733574
Louvain completed 21 runs in 17.428818464279175 seconds
Similarity is 0.8994125; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 1.2155718803405762 seconds
MEMORY 13.792612352
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00697723
After 2 runs, maximum modularity is Q = 0.00697724
Louvain completed 22 runs in 19.159449100494385 seconds
Similarity is 0.79851484; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.3088948726654053 seconds
MEMORY 13.792612352
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0054777
After 2 runs, maximum modularity is Q = 0.00637933
Louvain completed 22 runs in 16.390249729156494 seconds
Similarity is 0.8410438; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.3019249439239502 seconds
MEMORY 13.792612352
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00427911
After 2 runs, maximum modularity is Q = 0.00439068
After 4 runs, maximum modularity is Q = 0.00440417
Louvain completed 24 runs in 19.22598385810852 seconds
Similarity is 0.88598204; is_dissimilar is False
Got 2 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 1.1355187892913818 seconds
MEMORY 13.74091264
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00990382
After 2 runs, maximum modularity is Q = 0.00990383
Louvain completed 22 runs in 18.056727647781372 seconds
Similarity is 0.85944176; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.8615725040435791 seconds
MEMORY 13.74091264
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0185604
Louvain completed 21 runs in 16.87057590484619 seconds
Similarity is 0.8311509; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.47755956649780273 seconds
MEMORY 13.74091264
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00444484
Louvain completed 21 runs in 17.867013692855835 seconds
Similarity is 0.87520206; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.5008163452148438 seconds
MEMORY 13.74091264
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0167757
Louvain completed 21 runs in 16.36958074569702 seconds
Similarity is 0.913223; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.6701395511627197 seconds
MEMORY 13.74091264
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00568533
After 3 runs, maximum modularity is Q = 0.00611416
After 4 runs, maximum modularity is Q = 0.00611956
After 12 runs, maximum modularity is Q = 0.00612232
Louvain completed 32 runs in 27.628106355667114 seconds
Similarity is 0.86466414; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.3400533199310303 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0126803
Louvain completed 21 runs in 16.798762559890747 seconds
Similarity is 0.6034484; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.14386773109436035 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0148422
Louvain completed 21 runs in 15.376906156539917 seconds
Similarity is 0.49824274; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.2145531177520752 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00979133
Louvain completed 21 runs in 15.938128471374512 seconds
Similarity is 0.7196858; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.02468395233154297 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00569521
After 5 runs, maximum modularity is Q = 0.00577104
Louvain completed 25 runs in 19.072674989700317 seconds
Similarity is 0.7600992; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.013450145721435547 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00409699
After 2 runs, maximum modularity is Q = 0.0043624
After 3 runs, maximum modularity is Q = 0.00476095
After 7 runs, maximum modularity is Q = 0.0049405
Louvain completed 27 runs in 23.54065704345703 seconds
Similarity is 0.7136414; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.008455276489257812 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00620169
After 3 runs, maximum modularity is Q = 0.00628582
Louvain completed 23 runs in 17.651132822036743 seconds
Similarity is 0.8443024; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.02291274070739746 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00372796
After 2 runs, maximum modularity is Q = 0.00381008
Louvain completed 22 runs in 16.873881101608276 seconds
Similarity is 0.9270509; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.04235243797302246 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0124428
After 3 runs, maximum modularity is Q = 0.0124549
Louvain completed 23 runs in 17.3434476852417 seconds
Similarity is 0.864118; is_dissimilar is False
Got 6 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 0.12813687324523926 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0100902
Louvain completed 21 runs in 15.149794101715088 seconds
Similarity is 0.6885791; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.05207467079162598 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0039414
After 2 runs, maximum modularity is Q = 0.00455923
After 7 runs, maximum modularity is Q = 0.00463545
Louvain completed 27 runs in 21.131683826446533 seconds
Similarity is 0.8942932; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.013509750366210938 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0154124
Louvain completed 21 runs in 14.97204327583313 seconds
Similarity is 0.8473745; is_dissimilar is False
Got 2 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 0.11542177200317383 seconds
MEMORY 13.740916736
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0125176
Louvain completed 21 runs in 15.647984504699707 seconds
Similarity is 0.74810433; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.05437874794006348 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0132504
Louvain completed 21 runs in 15.37044906616211 seconds
Similarity is 0.7806651; is_dissimilar is True
Inspecting for spurious merging
Wrote graph to binary file in 0.17824149131774902 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00549504
Louvain completed 21 runs in 16.621508359909058 seconds
Similarity is 0.81753504; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.012847661972045898 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0144228
After 2 runs, maximum modularity is Q = 0.0144318
Louvain completed 22 runs in 15.935197591781616 seconds
Similarity is 0.8495423; is_dissimilar is False
Got 3 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 0.054437875747680664 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.00922513
Louvain completed 21 runs in 15.269986867904663 seconds
Similarity is 0.83493507; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.02700018882751465 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0193431
Louvain completed 21 runs in 15.134574890136719 seconds
Similarity is 0.90042657; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.02692556381225586 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0130489
After 2 runs, maximum modularity is Q = 0.0168431
Louvain completed 22 runs in 16.447123289108276 seconds
Similarity is 0.8715037; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.013255119323730469 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0146273
After 2 runs, maximum modularity is Q = 0.0349524
Louvain completed 22 runs in 16.494072437286377 seconds
Similarity is 0.52975154; is_dissimilar is True
Got 2 subclusters
Inspecting for spurious merging
Wrote graph to binary file in 0.011997461318969727 seconds
MEMORY 13.740920832
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0177272
Louvain completed 21 runs in 17.672269105911255 seconds
Similarity is 0.9128522; is_dissimilar is False
Inspecting for spurious merging
Wrote graph to binary file in 0.010609149932861328 seconds
MEMORY 13.657624576
Running Louvain modularity optimization
After 1 runs, maximum modularity is Q = 0.0326612
After 2 runs, maximum modularity is Q = 0.0327666
Louvain completed 22 runs in 16.646944761276245 seconds
Similarity is 0.61184216; is_dissimilar is True
Got 2 subclusters
Merging on 51 clusters
MEMORY 13.657624576
On merging iteration 1
Numbers for each pattern pre-subsample: [4535, 3412, 3044, 2545, 2337, 2144, 1981, 1912, 1025, 837, 1812, 1713, 1661, 1398, 1328, 1092, 977, 412, 409, 782, 667, 508, 511, 472, 46, 151, 108, 62, 34, 34, 168, 80, 138, 79, 34, 170, 118, 118, 40, 40, 76, 37, 34, 46, 44, 17, 20, 16, 11, 9, 10]
Numbers after subsampling: [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 46, 151, 108, 62, 34, 34, 168, 80, 138, 79, 34, 170, 118, 118, 40, 40, 76, 37, 34, 46, 44, 17, 20, 16, 11, 9, 10]
Cluster sizes
[4535 3412 3044 2545 2337 2144 1981 1912 1025 837 1812 1713 1661 1398
1328 1092 977 412 409 782 667 508 511 472 46 151 108 62
34 34 168 80 138 79 34 170 118 118 40 40 76 37
34 46 44 17 20 16 11 9 10]
Cross-contamination matrix:
[[1. 0.67 0.01 ... 0.05 0.09 0.01]
[0.73 1. 0.03 ... 0.06 0.04 0.02]
[0.08 0.11 1. ... 0.1 0.16 0.51]
...
[0. 0. 0. ... 1. 0. 0. ]
[0. 0. 0. ... 0. 1. 0. ]
[0. 0. 0. ... 0. 0. 1. ]]
Pattern-to-pattern sim matrix:
[[1. 0.82 0.11 ... 0.37 0.29 0.12]
[0.82 1. 0.21 ... 0.39 0.31 0.19]
[0.11 0.21 1. ... 0.2 0.28 0.42]
...
[0.39 0.41 0.21 ... 1. 0.2 0.23]
[0.3 0.35 0.28 ... 0.2 1. 0.16]
[0.13 0.21 0.42 ... 0.24 0.16 1. ]]
Collapsing 0 & 32 with crosscontam 0.935315618357488 and sim 0.9644776816770225
Collapsing 1 & 22 with crosscontam 0.7439919150617285 and sim 0.9015113876542068
Collapsing 1 & 32 with crosscontam 0.8395530428126271 and sim 0.9000446419643353
Collapsing 5 & 28 with crosscontam 0.8875731057398738 and sim 0.8900539593410017
Collapsing 10 & 21 with crosscontam 0.7261318312345679 and sim 0.8759707915253263
Collapsing 7 & 32 with crosscontam 0.722864054749388 and sim 0.8717348540396053
Collapsing 6 & 10 with crosscontam 0.7507281966666667 and sim 0.8620943508535983
Collapsing 22 & 34 with crosscontam 0.6206372754860499 and sim 0.8620724275442557
Collapsing 10 & 15 with crosscontam 0.7340165951234568 and sim 0.8620617408222342
Collapsing 1 & 7 with crosscontam 0.7630233908641976 and sim 0.861149841684182
Collapsing 10 & 11 with crosscontam 0.6840879422222224 and sim 0.8559102885032961
Aborting collapse as 11 & 21 have cross-contam 0.09513696543209893 and sim 0.4481399034518663
Collapsing 22 & 32 with crosscontam 0.6840055933977456 and sim 0.8509872840240437
Trimming eliminated 0 seqlets out of 4673
Removed 47 duplicate seqlets
Trimming eliminated 1 seqlets out of 3923
Removed 10 duplicate seqlets
Trimming eliminated 0 seqlets out of 8538
Skipped 6 seqlets
Removed 44 duplicate seqlets
Trimming eliminated 0 seqlets out of 2178
Removed 12 duplicate seqlets
Trimming eliminated 0 seqlets out of 2320
Removed 8 duplicate seqlets
Trimming eliminated 0 seqlets out of 10400
Removed 23 duplicate seqlets
Removed 3 duplicate seqlets
Trimming eliminated 0 seqlets out of 4293
Skipped 1 seqlets
Removed 19 duplicate seqlets
Trimming eliminated 0 seqlets out of 10408
Removed 36 duplicate seqlets
Trimming eliminated 0 seqlets out of 5365
Skipped 1 seqlets
Removed 19 duplicate seqlets
Removed 4 duplicate seqlets
Unmerged patterns remapping: OrderedDict([(2, 2), (3, 3), (4, 4), (8, 10), (9, 12), (11, 6), (12, 7), (13, 8), (14, 9), (16, 11), (17, 16), (18, 17), (19, 13), (20, 14), (23, 15), (24, 28), (25, 20), (26, 23), (27, 27), (29, 34), (30, 19), (31, 24), (33, 25), (35, 18), (36, 21), (37, 22), (38, 31), (39, 32), (40, 26), (41, 33), (42, 35), (43, 29), (44, 30), (45, 37), (46, 36), (47, 38), (48, 39), (49, 41), (50, 40)])
Time spent on merging iteration: 1416.5477232933044
On merging iteration 2
Numbers for each pattern pre-subsample: [10372, 5341, 3044, 2545, 2337, 2166, 1713, 1661, 1398, 1328, 1025, 977, 837, 782, 667, 472, 412, 409, 170, 168, 151, 118, 118, 108, 80, 79, 76, 62, 46, 46, 44, 40, 40, 37, 34, 34, 20, 17, 16, 11, 10, 9]
Numbers after subsampling: [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 170, 168, 151, 118, 118, 108, 80, 79, 76, 62, 46, 46, 44, 40, 40, 37, 34, 34, 20, 17, 16, 11, 10, 9]
Cluster sizes
[10372 5341 3044 2545 2337 2166 1713 1661 1398 1328 1025 977
837 782 667 472 412 409 170 168 151 118 118 108
80 79 76 62 46 46 44 40 40 37 34 34
20 17 16 11 10 9]
Cross-contamination matrix:
[[1. 0.57 0.52 ... 0.62 0.51 0.49]
[0.48 1. 0.35 ... 1. 0.61 0.82]
[0.09 0.07 1. ... 0.1 0.51 0.16]
...
[0. 0. 0. ... 1. 0. 0. ]
[0. 0. 0. ... 0. 1. 0. ]
[0. 0. 0. ... 0. 0. 1. ]]
Pattern-to-pattern sim matrix:
[[1. 0.45 0.16 ... 0.4 0.15 0.24]
[0.45 1. 0.16 ... 0.71 0.31 0.58]
[0.16 0.16 1. ... 0.2 0.42 0.28]
...
[0.41 0.74 0.21 ... 1. 0.23 0.2 ]
[0.16 0.34 0.42 ... 0.24 1. 0.16]
[0.27 0.65 0.28 ... 0.2 0.16 1. ]]
Collapsing 1 & 6 with crosscontam 0.7592684666666666 and sim 0.8654233820389916
Collapsing 1 & 28 with crosscontam 0.8554507093504287 and sim 0.8401876731063171
Trimming eliminated 0 seqlets out of 7054
Skipped 2 seqlets
Removed 7 duplicate seqlets
Removed 5 duplicate seqlets
Trimming eliminated 0 seqlets out of 7086
Removed 4 duplicate seqlets
Unmerged patterns remapping: OrderedDict([(0, 0), (2, 2), (3, 3), (4, 4), (5, 5), (7, 6), (8, 7), (9, 8), (10, 9), (11, 10), (12, 11), (13, 12), (14, 13), (15, 14), (16, 15), (17, 16), (18, 17), (19, 18), (20, 19), (21, 20), (22, 21), (23, 22), (24, 23), (25, 24), (26, 25), (27, 26), (29, 27), (30, 28), (31, 29), (32, 30), (33, 31), (34, 32), (35, 33), (36, 34), (37, 35), (38, 36), (39, 37), (40, 38), (41, 39)])
Time spent on merging iteration: 193.02080368995667
On merging iteration 3
Numbers for each pattern pre-subsample: [10372, 7082, 3044, 2545, 2337, 2166, 1661, 1398, 1328, 1025, 977, 837, 782, 667, 472, 412, 409, 170, 168, 151, 118, 118, 108, 80, 79, 76, 62, 46, 44, 40, 40, 37, 34, 34, 20, 17, 16, 11, 10, 9]
Numbers after subsampling: [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 170, 168, 151, 118, 118, 108, 80, 79, 76, 62, 46, 44, 40, 40, 37, 34, 34, 20, 17, 16, 11, 10, 9]
Cluster sizes
[10372 7082 3044 2545 2337 2166 1661 1398 1328 1025 977 837
782 667 472 412 409 170 168 151 118 118 108 80
79 76 62 46 44 40 40 37 34 34 20 17
16 11 10 9]
Cross-contamination matrix:
[[1. 0.51 0.52 ... 0.62 0.51 0.49]
[0.79 1. 0.72 ... 1. 0.98 1. ]
[0.09 0.06 1. ... 0.1 0.51 0.16]
...
[0. 0. 0. ... 1. 0. 0. ]
[0. 0. 0. ... 0. 1. 0. ]
[0. 0. 0. ... 0. 0. 1. ]]
Pattern-to-pattern sim matrix:
[[1. 0.46 0.16 ... 0.4 0.15 0.24]
[0.46 1. 0.16 ... 0.71 0.31 0.56]
[0.16 0.16 1. ... 0.2 0.42 0.28]
...
[0.41 0.73 0.21 ... 1. 0.23 0.2 ]
[0.16 0.33 0.42 ... 0.24 1. 0.16]
[0.27 0.63 0.28 ... 0.2 0.16 1. ]]
Got 40 patterns after merging
MEMORY 13.659529216
Performing seqlet reassignment
MEMORY 13.659529216
Cross contin jaccard time taken: 0.54 s
Cross contin jaccard time taken: 0.5 s
Discarded 10 seqlets
Removed 1 duplicate seqlets
Removed 1 duplicate seqlets
Removed 15 duplicate seqlets
Removed 1 duplicate seqlets
Removed 1 duplicate seqlets
Removed 9 duplicate seqlets
Got 27 patterns after reassignment
MEMORY 13.659660288
Total time taken is 10461.84s
MEMORY 13.659660288
Applying subclustering to the final motifs
On pattern 0
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.7s [Parallel(n_jobs=10)]: Done 340 tasks | elapsed: 6.5s [Parallel(n_jobs=10)]: Done 840 tasks | elapsed: 16.2s [Parallel(n_jobs=10)]: Done 1540 tasks | elapsed: 29.1s [Parallel(n_jobs=10)]: Done 2440 tasks | elapsed: 45.4s [Parallel(n_jobs=10)]: Done 3540 tasks | elapsed: 1.1min [Parallel(n_jobs=10)]: Done 4840 tasks | elapsed: 1.5min [Parallel(n_jobs=10)]: Done 6340 tasks | elapsed: 1.9min [Parallel(n_jobs=10)]: Done 8040 tasks | elapsed: 2.4min [Parallel(n_jobs=10)]: Done 9940 tasks | elapsed: 3.0min [Parallel(n_jobs=10)]: Done 10430 out of 10449 | elapsed: 3.1min remaining: 0.3s [Parallel(n_jobs=10)]: Done 10449 out of 10449 | elapsed: 3.1min finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 10449 samples in 0.053s... [t-SNE] Computed neighbors for 10449 samples in 0.007s... [t-SNE] Computed conditional probabilities for sample 1000 / 10449
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computed conditional probabilities for sample 2000 / 10449 [t-SNE] Computed conditional probabilities for sample 3000 / 10449 [t-SNE] Computed conditional probabilities for sample 4000 / 10449 [t-SNE] Computed conditional probabilities for sample 5000 / 10449 [t-SNE] Computed conditional probabilities for sample 6000 / 10449 [t-SNE] Computed conditional probabilities for sample 7000 / 10449 [t-SNE] Computed conditional probabilities for sample 8000 / 10449 [t-SNE] Computed conditional probabilities for sample 9000 / 10449 [t-SNE] Computed conditional probabilities for sample 10000 / 10449 [t-SNE] Computed conditional probabilities for sample 10449 / 10449 [t-SNE] Mean sigma: 0.248145 [t-SNE] Computed conditional probabilities in 0.775s [t-SNE] Iteration 50: error = 95.4026794, gradient norm = 0.0000156 (50 iterations in 9.048s) [t-SNE] Iteration 100: error = 95.4024200, gradient norm = 0.0000467 (50 iterations in 8.727s) [t-SNE] Iteration 150: error = 95.3300247, gradient norm = 0.0007519 (50 iterations in 7.722s) [t-SNE] Iteration 200: error = 95.1420593, gradient norm = 0.0000520 (50 iterations in 6.126s) [t-SNE] Iteration 250: error = 95.1367035, gradient norm = 0.0000072 (50 iterations in 5.756s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 95.136703 [t-SNE] Iteration 300: error = 4.2174821, gradient norm = 0.0017327 (50 iterations in 6.979s) [t-SNE] Iteration 350: error = 3.7858899, gradient norm = 0.0006329 (50 iterations in 7.097s) [t-SNE] Iteration 400: error = 3.5975933, gradient norm = 0.0003595 (50 iterations in 6.645s) [t-SNE] Iteration 450: error = 3.4877884, gradient norm = 0.0002413 (50 iterations in 6.588s) [t-SNE] Iteration 500: error = 3.4137485, gradient norm = 0.0001772 (50 iterations in 6.584s) [t-SNE] Iteration 550: error = 3.3603568, gradient norm = 0.0001364 (50 iterations in 6.370s) [t-SNE] Iteration 600: error = 3.3204238, gradient norm = 0.0001146 (50 iterations in 6.383s) [t-SNE] Iteration 650: error = 3.2897985, gradient norm = 0.0000945 (50 iterations in 6.492s) [t-SNE] Iteration 700: error = 3.2655895, gradient norm = 0.0000826 (50 iterations in 6.539s) [t-SNE] Iteration 750: error = 3.2456822, gradient norm = 0.0000771 (50 iterations in 6.573s) [t-SNE] Iteration 800: error = 3.2301733, gradient norm = 0.0000713 (50 iterations in 6.505s) [t-SNE] Iteration 850: error = 3.2179275, gradient norm = 0.0000643 (50 iterations in 6.473s) [t-SNE] Iteration 900: error = 3.2080212, gradient norm = 0.0000593 (50 iterations in 6.648s) [t-SNE] Iteration 950: error = 3.1992910, gradient norm = 0.0000529 (50 iterations in 6.786s) [t-SNE] Iteration 1000: error = 3.1914346, gradient norm = 0.0000489 (50 iterations in 6.603s) [t-SNE] KL divergence after 1000 iterations: 3.191435 [t-SNE] Computed conditional probabilities for sample 1000 / 10449 [t-SNE] Computed conditional probabilities for sample 2000 / 10449 [t-SNE] Computed conditional probabilities for sample 3000 / 10449 [t-SNE] Computed conditional probabilities for sample 4000 / 10449 [t-SNE] Computed conditional probabilities for sample 5000 / 10449 [t-SNE] Computed conditional probabilities for sample 6000 / 10449 [t-SNE] Computed conditional probabilities for sample 7000 / 10449 [t-SNE] Computed conditional probabilities for sample 8000 / 10449 [t-SNE] Computed conditional probabilities for sample 9000 / 10449 [t-SNE] Computed conditional probabilities for sample 10000 / 10449 [t-SNE] Computed conditional probabilities for sample 10449 / 10449 [t-SNE] Mean sigma: 0.248145 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.5min
Quality: 0.5789882078336033 Quality: 0.5797438093339089 Quality: 0.5801635406374814 Quality: 0.5801890578558699
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 3.9min finished
On pattern 1
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.7s [Parallel(n_jobs=10)]: Done 340 tasks | elapsed: 4.0s [Parallel(n_jobs=10)]: Done 840 tasks | elapsed: 9.3s [Parallel(n_jobs=10)]: Done 1540 tasks | elapsed: 17.0s [Parallel(n_jobs=10)]: Done 2440 tasks | elapsed: 27.0s [Parallel(n_jobs=10)]: Done 3540 tasks | elapsed: 39.3s [Parallel(n_jobs=10)]: Done 4840 tasks | elapsed: 54.4s [Parallel(n_jobs=10)]: Done 6340 tasks | elapsed: 1.2min [Parallel(n_jobs=10)]: Done 7098 out of 7117 | elapsed: 1.3min remaining: 0.2s [Parallel(n_jobs=10)]: Done 7117 out of 7117 | elapsed: 1.3min finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 7117 samples in 0.035s... [t-SNE] Computed neighbors for 7117 samples in 0.005s... [t-SNE] Computed conditional probabilities for sample 1000 / 7117 [t-SNE] Computed conditional probabilities for sample 2000 / 7117
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computed conditional probabilities for sample 3000 / 7117 [t-SNE] Computed conditional probabilities for sample 4000 / 7117 [t-SNE] Computed conditional probabilities for sample 5000 / 7117 [t-SNE] Computed conditional probabilities for sample 6000 / 7117 [t-SNE] Computed conditional probabilities for sample 7000 / 7117 [t-SNE] Computed conditional probabilities for sample 7117 / 7117 [t-SNE] Mean sigma: 0.272219 [t-SNE] Computed conditional probabilities in 0.490s [t-SNE] Iteration 50: error = 91.2811050, gradient norm = 0.0000634 (50 iterations in 5.098s) [t-SNE] Iteration 100: error = 89.6503067, gradient norm = 0.0127861 (50 iterations in 10.215s) [t-SNE] Iteration 150: error = 89.1141357, gradient norm = 0.0000144 (50 iterations in 5.252s) [t-SNE] Iteration 200: error = 89.1128464, gradient norm = 0.0000080 (50 iterations in 4.300s) [t-SNE] Iteration 250: error = 89.1125031, gradient norm = 0.0000152 (50 iterations in 4.229s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 89.112503 [t-SNE] Iteration 300: error = 3.5561519, gradient norm = 0.0014822 (50 iterations in 6.105s) [t-SNE] Iteration 350: error = 3.2249932, gradient norm = 0.0005413 (50 iterations in 4.414s) [t-SNE] Iteration 400: error = 3.0753443, gradient norm = 0.0003126 (50 iterations in 4.312s) [t-SNE] Iteration 450: error = 2.9877033, gradient norm = 0.0002113 (50 iterations in 4.271s) [t-SNE] Iteration 500: error = 2.9292862, gradient norm = 0.0001616 (50 iterations in 4.235s) [t-SNE] Iteration 550: error = 2.8883181, gradient norm = 0.0001284 (50 iterations in 4.165s) [t-SNE] Iteration 600: error = 2.8587439, gradient norm = 0.0001065 (50 iterations in 4.054s) [t-SNE] Iteration 650: error = 2.8372159, gradient norm = 0.0000885 (50 iterations in 4.109s) [t-SNE] Iteration 700: error = 2.8199861, gradient norm = 0.0000767 (50 iterations in 4.111s) [t-SNE] Iteration 750: error = 2.8062387, gradient norm = 0.0000691 (50 iterations in 4.214s) [t-SNE] Iteration 800: error = 2.7952132, gradient norm = 0.0000610 (50 iterations in 4.182s) [t-SNE] Iteration 850: error = 2.7861021, gradient norm = 0.0000564 (50 iterations in 4.312s) [t-SNE] Iteration 900: error = 2.7784746, gradient norm = 0.0000539 (50 iterations in 4.236s) [t-SNE] Iteration 950: error = 2.7726045, gradient norm = 0.0000488 (50 iterations in 4.135s) [t-SNE] Iteration 1000: error = 2.7676487, gradient norm = 0.0000479 (50 iterations in 4.250s) [t-SNE] KL divergence after 1000 iterations: 2.767649 [t-SNE] Computed conditional probabilities for sample 1000 / 7117 [t-SNE] Computed conditional probabilities for sample 2000 / 7117 [t-SNE] Computed conditional probabilities for sample 3000 / 7117 [t-SNE] Computed conditional probabilities for sample 4000 / 7117 [t-SNE] Computed conditional probabilities for sample 5000 / 7117 [t-SNE] Computed conditional probabilities for sample 6000 / 7117 [t-SNE] Computed conditional probabilities for sample 7000 / 7117 [t-SNE] Computed conditional probabilities for sample 7117 / 7117 [t-SNE] Mean sigma: 0.272219 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 1.4min
Quality: 0.6214561421350129 Quality: 0.6221664749790374 Quality: 0.6224414691244131 Quality: 0.6224986813904207 Quality: 0.6226435031373709 Quality: 0.6227290667304873 Quality: 0.6232977125162873
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 2.2min finished
On pattern 2
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.3s [Parallel(n_jobs=10)]: Done 620 tasks | elapsed: 2.9s [Parallel(n_jobs=10)]: Done 1620 tasks | elapsed: 8.4s [Parallel(n_jobs=10)]: Done 3008 tasks | elapsed: 16.1s [Parallel(n_jobs=10)]: Done 3051 out of 3051 | elapsed: 16.3s finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 3051 samples in 0.016s... [t-SNE] Computed neighbors for 3051 samples in 0.003s... [t-SNE] Computed conditional probabilities for sample 1000 / 3051 [t-SNE] Computed conditional probabilities for sample 2000 / 3051
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computed conditional probabilities for sample 3000 / 3051 [t-SNE] Computed conditional probabilities for sample 3051 / 3051 [t-SNE] Mean sigma: 0.273515 [t-SNE] Computed conditional probabilities in 0.207s [t-SNE] Iteration 50: error = 80.9520340, gradient norm = 0.0127325 (50 iterations in 5.495s) [t-SNE] Iteration 100: error = 80.9514618, gradient norm = 0.0091614 (50 iterations in 4.758s) [t-SNE] Iteration 150: error = 80.9516983, gradient norm = 0.0106412 (50 iterations in 5.646s) [t-SNE] Iteration 200: error = 80.9527359, gradient norm = 0.0153602 (50 iterations in 4.060s) [t-SNE] Iteration 250: error = 80.9523773, gradient norm = 0.0140144 (50 iterations in 3.607s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 80.952377 [t-SNE] Iteration 300: error = 2.7279625, gradient norm = 0.0016802 (50 iterations in 2.014s) [t-SNE] Iteration 350: error = 2.4670014, gradient norm = 0.0005437 (50 iterations in 1.552s) [t-SNE] Iteration 400: error = 2.3624761, gradient norm = 0.0002949 (50 iterations in 1.520s) [t-SNE] Iteration 450: error = 2.3109052, gradient norm = 0.0002051 (50 iterations in 1.543s) [t-SNE] Iteration 500: error = 2.2845378, gradient norm = 0.0001396 (50 iterations in 1.550s) [t-SNE] Iteration 550: error = 2.2676892, gradient norm = 0.0001226 (50 iterations in 1.557s) [t-SNE] Iteration 600: error = 2.2566888, gradient norm = 0.0000943 (50 iterations in 1.579s) [t-SNE] Iteration 650: error = 2.2491477, gradient norm = 0.0000800 (50 iterations in 1.570s) [t-SNE] Iteration 700: error = 2.2435489, gradient norm = 0.0000763 (50 iterations in 1.510s) [t-SNE] Iteration 750: error = 2.2391670, gradient norm = 0.0000688 (50 iterations in 1.599s) [t-SNE] Iteration 800: error = 2.2359583, gradient norm = 0.0000590 (50 iterations in 1.541s) [t-SNE] Iteration 850: error = 2.2334111, gradient norm = 0.0000575 (50 iterations in 1.587s) [t-SNE] Iteration 900: error = 2.2314558, gradient norm = 0.0000577 (50 iterations in 1.614s) [t-SNE] Iteration 950: error = 2.2301366, gradient norm = 0.0000522 (50 iterations in 1.570s) [t-SNE] Iteration 1000: error = 2.2287884, gradient norm = 0.0000580 (50 iterations in 1.557s) [t-SNE] KL divergence after 1000 iterations: 2.228788 [t-SNE] Computed conditional probabilities for sample 1000 / 3051 [t-SNE] Computed conditional probabilities for sample 2000 / 3051 [t-SNE] Computed conditional probabilities for sample 3000 / 3051 [t-SNE] Computed conditional probabilities for sample 3051 / 3051 [t-SNE] Mean sigma: 0.273515 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 21.6s
Quality: 0.5745324610276767 Quality: 0.5751460000808041 Quality: 0.5751507381451649
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 35.6s finished
On pattern 3
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.2s [Parallel(n_jobs=10)]: Done 620 tasks | elapsed: 2.6s [Parallel(n_jobs=10)]: Done 1620 tasks | elapsed: 7.4s [Parallel(n_jobs=10)]: Done 2581 out of 2581 | elapsed: 11.6s finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 2581 samples in 0.012s... [t-SNE] Computed neighbors for 2581 samples in 0.002s... [t-SNE] Computed conditional probabilities for sample 1000 / 2581 [t-SNE] Computed conditional probabilities for sample 2000 / 2581 [t-SNE] Computed conditional probabilities for sample 2581 / 2581 [t-SNE] Mean sigma: 0.266135 [t-SNE] Computed conditional probabilities in 0.168s
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 79.2176514, gradient norm = 0.0972788 (50 iterations in 3.262s) [t-SNE] Iteration 100: error = 79.1868057, gradient norm = 0.0880403 (50 iterations in 3.368s) [t-SNE] Iteration 150: error = 78.9401703, gradient norm = 0.1015745 (50 iterations in 3.474s) [t-SNE] Iteration 200: error = 79.1182175, gradient norm = 0.0919601 (50 iterations in 3.158s) [t-SNE] Iteration 250: error = 79.0734558, gradient norm = 0.0960414 (50 iterations in 2.665s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 79.073456 [t-SNE] Iteration 300: error = 2.7734911, gradient norm = 0.0012670 (50 iterations in 1.443s) [t-SNE] Iteration 350: error = 2.6096685, gradient norm = 0.0005049 (50 iterations in 1.239s) [t-SNE] Iteration 400: error = 2.5395341, gradient norm = 0.0002673 (50 iterations in 1.240s) [t-SNE] Iteration 450: error = 2.5027959, gradient norm = 0.0001634 (50 iterations in 1.216s) [t-SNE] Iteration 500: error = 2.4821000, gradient norm = 0.0001376 (50 iterations in 1.230s) [t-SNE] Iteration 550: error = 2.4698846, gradient norm = 0.0001142 (50 iterations in 1.257s) [t-SNE] Iteration 600: error = 2.4629936, gradient norm = 0.0001047 (50 iterations in 1.296s) [t-SNE] Iteration 650: error = 2.4577763, gradient norm = 0.0000957 (50 iterations in 1.264s) [t-SNE] Iteration 700: error = 2.4530308, gradient norm = 0.0000823 (50 iterations in 1.264s) [t-SNE] Iteration 750: error = 2.4476111, gradient norm = 0.0000821 (50 iterations in 1.229s) [t-SNE] Iteration 800: error = 2.4447322, gradient norm = 0.0000683 (50 iterations in 1.236s) [t-SNE] Iteration 850: error = 2.4425249, gradient norm = 0.0000679 (50 iterations in 1.280s) [t-SNE] Iteration 900: error = 2.4409590, gradient norm = 0.0000585 (50 iterations in 1.257s) [t-SNE] Iteration 950: error = 2.4395792, gradient norm = 0.0000514 (50 iterations in 1.269s) [t-SNE] Iteration 1000: error = 2.4380527, gradient norm = 0.0000516 (50 iterations in 1.269s) [t-SNE] KL divergence after 1000 iterations: 2.438053 [t-SNE] Computed conditional probabilities for sample 1000 / 2581 [t-SNE] Computed conditional probabilities for sample 2000 / 2581 [t-SNE] Computed conditional probabilities for sample 2581 / 2581 [t-SNE] Mean sigma: 0.266135 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 19.1s
Quality: 0.5153294790975234 Quality: 0.5179601791932023 Quality: 0.5182040584352147
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 31.2s finished
On pattern 4
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.2s [Parallel(n_jobs=10)]: Done 620 tasks | elapsed: 2.3s [Parallel(n_jobs=10)]: Done 1620 tasks | elapsed: 5.9s [Parallel(n_jobs=10)]: Done 2334 out of 2353 | elapsed: 8.8s remaining: 0.1s [Parallel(n_jobs=10)]: Done 2353 out of 2353 | elapsed: 8.8s finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 2353 samples in 0.010s... [t-SNE] Computed neighbors for 2353 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 1000 / 2353 [t-SNE] Computed conditional probabilities for sample 2000 / 2353 [t-SNE] Computed conditional probabilities for sample 2353 / 2353 [t-SNE] Mean sigma: 0.268039 [t-SNE] Computed conditional probabilities in 0.156s
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 78.4455490, gradient norm = 0.1274652 (50 iterations in 2.109s) [t-SNE] Iteration 100: error = 78.4364395, gradient norm = 0.1109584 (50 iterations in 1.987s) [t-SNE] Iteration 150: error = 78.4814377, gradient norm = 0.1035236 (50 iterations in 2.712s) [t-SNE] Iteration 200: error = 78.8519363, gradient norm = 0.1110280 (50 iterations in 2.788s) [t-SNE] Iteration 250: error = 78.6199646, gradient norm = 0.1051478 (50 iterations in 3.305s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 78.619965 [t-SNE] Iteration 300: error = 2.8527775, gradient norm = 0.0014622 (50 iterations in 1.741s) [t-SNE] Iteration 350: error = 2.6910357, gradient norm = 0.0005693 (50 iterations in 1.117s) [t-SNE] Iteration 400: error = 2.6187816, gradient norm = 0.0003028 (50 iterations in 1.086s) [t-SNE] Iteration 450: error = 2.5814548, gradient norm = 0.0002000 (50 iterations in 1.084s) [t-SNE] Iteration 500: error = 2.5602551, gradient norm = 0.0001519 (50 iterations in 1.151s) [t-SNE] Iteration 550: error = 2.5473728, gradient norm = 0.0001247 (50 iterations in 1.159s) [t-SNE] Iteration 600: error = 2.5387759, gradient norm = 0.0001087 (50 iterations in 1.130s) [t-SNE] Iteration 650: error = 2.5328405, gradient norm = 0.0000937 (50 iterations in 1.145s) [t-SNE] Iteration 700: error = 2.5284679, gradient norm = 0.0001035 (50 iterations in 1.154s) [t-SNE] Iteration 750: error = 2.5258651, gradient norm = 0.0000662 (50 iterations in 1.118s) [t-SNE] Iteration 800: error = 2.5232363, gradient norm = 0.0000616 (50 iterations in 1.118s) [t-SNE] Iteration 850: error = 2.5210183, gradient norm = 0.0000823 (50 iterations in 1.137s) [t-SNE] Iteration 900: error = 2.5192871, gradient norm = 0.0000618 (50 iterations in 1.170s) [t-SNE] Iteration 950: error = 2.5181091, gradient norm = 0.0000564 (50 iterations in 1.167s) [t-SNE] Iteration 1000: error = 2.5169547, gradient norm = 0.0000590 (50 iterations in 1.179s) [t-SNE] KL divergence after 1000 iterations: 2.516955 [t-SNE] Computed conditional probabilities for sample 1000 / 2353 [t-SNE] Computed conditional probabilities for sample 2000 / 2353 [t-SNE] Computed conditional probabilities for sample 2353 / 2353 [t-SNE] Mean sigma: 0.268039 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 17.0s
Quality: 0.4848020650483093 Quality: 0.48684440362746895 Quality: 0.4868771366174969 Quality: 0.4869080007555646 Quality: 0.48697854954861336
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 27.1s finished
On pattern 5
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.2s [Parallel(n_jobs=10)]: Done 1100 tasks | elapsed: 3.6s [Parallel(n_jobs=10)]: Done 2153 out of 2172 | elapsed: 7.0s remaining: 0.1s [Parallel(n_jobs=10)]: Done 2172 out of 2172 | elapsed: 7.0s finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 2172 samples in 0.010s... [t-SNE] Computed neighbors for 2172 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 1000 / 2172 [t-SNE] Computed conditional probabilities for sample 2000 / 2172 [t-SNE] Computed conditional probabilities for sample 2172 / 2172 [t-SNE] Mean sigma: 0.265223 [t-SNE] Computed conditional probabilities in 0.147s
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 77.8150635, gradient norm = 0.1324910 (50 iterations in 1.687s) [t-SNE] Iteration 100: error = 77.9089661, gradient norm = 0.1409067 (50 iterations in 1.739s) [t-SNE] Iteration 150: error = 78.6883316, gradient norm = 0.1003582 (50 iterations in 1.845s) [t-SNE] Iteration 200: error = 77.8246460, gradient norm = 0.1231222 (50 iterations in 1.625s) [t-SNE] Iteration 250: error = 78.0776291, gradient norm = 0.1219886 (50 iterations in 1.710s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 78.077629 [t-SNE] Iteration 300: error = 2.6375692, gradient norm = 0.0019356 (50 iterations in 1.563s) [t-SNE] Iteration 350: error = 2.5041394, gradient norm = 0.0004745 (50 iterations in 1.003s) [t-SNE] Iteration 400: error = 2.4441671, gradient norm = 0.0002770 (50 iterations in 1.006s) [t-SNE] Iteration 450: error = 2.4150715, gradient norm = 0.0001972 (50 iterations in 1.010s) [t-SNE] Iteration 500: error = 2.3985353, gradient norm = 0.0001337 (50 iterations in 1.011s) [t-SNE] Iteration 550: error = 2.3887336, gradient norm = 0.0001032 (50 iterations in 1.065s) [t-SNE] Iteration 600: error = 2.3820515, gradient norm = 0.0000801 (50 iterations in 1.045s) [t-SNE] Iteration 650: error = 2.3773105, gradient norm = 0.0000756 (50 iterations in 1.043s) [t-SNE] Iteration 700: error = 2.3739641, gradient norm = 0.0000685 (50 iterations in 1.073s) [t-SNE] Iteration 750: error = 2.3714709, gradient norm = 0.0000698 (50 iterations in 1.016s) [t-SNE] Iteration 800: error = 2.3697178, gradient norm = 0.0000557 (50 iterations in 1.028s) [t-SNE] Iteration 850: error = 2.3683693, gradient norm = 0.0000487 (50 iterations in 1.042s) [t-SNE] Iteration 900: error = 2.3669744, gradient norm = 0.0000496 (50 iterations in 1.069s) [t-SNE] Iteration 950: error = 2.3658988, gradient norm = 0.0000391 (50 iterations in 1.035s) [t-SNE] Iteration 1000: error = 2.3648868, gradient norm = 0.0000409 (50 iterations in 1.041s) [t-SNE] KL divergence after 1000 iterations: 2.364887 [t-SNE] Computed conditional probabilities for sample 1000 / 2172 [t-SNE] Computed conditional probabilities for sample 2000 / 2172 [t-SNE] Computed conditional probabilities for sample 2172 / 2172 [t-SNE] Mean sigma: 0.265223 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 15.4s
Quality: 0.5046317521889098 Quality: 0.5051494523879523 Quality: 0.5053826314233166 Quality: 0.5057363006433276 Quality: 0.5058702962736689 Quality: 0.5060758384188896
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 25.4s finished
On pattern 6
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 1100 tasks | elapsed: 2.7s [Parallel(n_jobs=10)]: Done 1665 out of 1665 | elapsed: 4.0s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 1665 samples in 0.008s... [t-SNE] Computed neighbors for 1665 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 1000 / 1665 [t-SNE] Computed conditional probabilities for sample 1665 / 1665 [t-SNE] Mean sigma: 0.268699 [t-SNE] Computed conditional probabilities in 0.116s [t-SNE] Iteration 50: error = 75.0960999, gradient norm = 0.2119861 (50 iterations in 1.219s) [t-SNE] Iteration 100: error = 75.0793762, gradient norm = 0.1909408 (50 iterations in 1.304s) [t-SNE] Iteration 150: error = 75.1263657, gradient norm = 0.1810585 (50 iterations in 1.432s) [t-SNE] Iteration 200: error = 75.4273453, gradient norm = 0.1869145 (50 iterations in 1.520s) [t-SNE] Iteration 250: error = 76.1043472, gradient norm = 0.1490247 (50 iterations in 1.329s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 76.104347 [t-SNE] Iteration 300: error = 2.5130334, gradient norm = 0.0015800 (50 iterations in 0.839s) [t-SNE] Iteration 350: error = 2.3974681, gradient norm = 0.0004965 (50 iterations in 0.729s) [t-SNE] Iteration 400: error = 2.3456299, gradient norm = 0.0003208 (50 iterations in 0.770s) [t-SNE] Iteration 450: error = 2.3228476, gradient norm = 0.0001796 (50 iterations in 0.762s) [t-SNE] Iteration 500: error = 2.3110669, gradient norm = 0.0001456 (50 iterations in 0.744s) [t-SNE] Iteration 550: error = 2.3045089, gradient norm = 0.0001079 (50 iterations in 0.740s) [t-SNE] Iteration 600: error = 2.2996244, gradient norm = 0.0000908 (50 iterations in 0.764s) [t-SNE] Iteration 650: error = 2.2952240, gradient norm = 0.0001130 (50 iterations in 0.774s) [t-SNE] Iteration 700: error = 2.2932675, gradient norm = 0.0000800 (50 iterations in 0.742s) [t-SNE] Iteration 750: error = 2.2915757, gradient norm = 0.0000588 (50 iterations in 0.742s) [t-SNE] Iteration 800: error = 2.2899115, gradient norm = 0.0000633 (50 iterations in 0.760s) [t-SNE] Iteration 850: error = 2.2887263, gradient norm = 0.0000550 (50 iterations in 0.785s) [t-SNE] Iteration 900: error = 2.2877710, gradient norm = 0.0000614 (50 iterations in 0.749s) [t-SNE] Iteration 950: error = 2.2874410, gradient norm = 0.0000548 (50 iterations in 0.783s) [t-SNE] Iteration 1000: error = 2.2867508, gradient norm = 0.0000532 (50 iterations in 0.782s) [t-SNE] KL divergence after 1000 iterations: 2.286751 [t-SNE] Computed conditional probabilities for sample 1000 / 1665 [t-SNE] Computed conditional probabilities for sample 1665 / 1665 [t-SNE] Mean sigma: 0.268699 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 12.3s
Quality: 0.4853340507246737 Quality: 0.4855522934213229 Quality: 0.48621620511589553 Quality: 0.48624187409595665
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 19.6s finished
On pattern 7
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 1100 tasks | elapsed: 2.2s [Parallel(n_jobs=10)]: Done 1399 out of 1399 | elapsed: 2.7s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 1399 samples in 0.007s... [t-SNE] Computed neighbors for 1399 samples in 0.002s... [t-SNE] Computed conditional probabilities for sample 1000 / 1399 [t-SNE] Computed conditional probabilities for sample 1399 / 1399 [t-SNE] Mean sigma: 0.257605 [t-SNE] Computed conditional probabilities in 0.097s [t-SNE] Iteration 50: error = 72.6122055, gradient norm = 0.2167766 (50 iterations in 0.958s) [t-SNE] Iteration 100: error = 72.4994736, gradient norm = 0.2062808 (50 iterations in 1.088s) [t-SNE] Iteration 150: error = 72.6476212, gradient norm = 0.2049416 (50 iterations in 1.096s) [t-SNE] Iteration 200: error = 73.2548294, gradient norm = 0.1890233 (50 iterations in 1.056s) [t-SNE] Iteration 250: error = 72.9196625, gradient norm = 0.2086316 (50 iterations in 1.054s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 72.919662 [t-SNE] Iteration 300: error = 2.0026572, gradient norm = 0.0020389 (50 iterations in 0.714s) [t-SNE] Iteration 350: error = 1.8884474, gradient norm = 0.0004482 (50 iterations in 0.605s) [t-SNE] Iteration 400: error = 1.8463651, gradient norm = 0.0002547 (50 iterations in 0.624s) [t-SNE] Iteration 450: error = 1.8281761, gradient norm = 0.0002026 (50 iterations in 0.626s) [t-SNE] Iteration 500: error = 1.8201379, gradient norm = 0.0001467 (50 iterations in 0.615s) [t-SNE] Iteration 550: error = 1.8152578, gradient norm = 0.0001354 (50 iterations in 0.609s) [t-SNE] Iteration 600: error = 1.8119612, gradient norm = 0.0001096 (50 iterations in 0.627s) [t-SNE] Iteration 650: error = 1.8098161, gradient norm = 0.0000814 (50 iterations in 0.632s) [t-SNE] Iteration 700: error = 1.8078405, gradient norm = 0.0000823 (50 iterations in 0.617s) [t-SNE] Iteration 750: error = 1.8066399, gradient norm = 0.0000878 (50 iterations in 0.590s) [t-SNE] Iteration 800: error = 1.8055239, gradient norm = 0.0000788 (50 iterations in 0.626s) [t-SNE] Iteration 850: error = 1.8040190, gradient norm = 0.0001048 (50 iterations in 0.620s) [t-SNE] Iteration 900: error = 1.8029851, gradient norm = 0.0000899 (50 iterations in 0.654s) [t-SNE] Iteration 950: error = 1.8021054, gradient norm = 0.0001242 (50 iterations in 0.596s) [t-SNE] Iteration 1000: error = 1.8025398, gradient norm = 0.0001128 (50 iterations in 0.641s) [t-SNE] KL divergence after 1000 iterations: 1.802540 [t-SNE] Computed conditional probabilities for sample 1000 / 1399 [t-SNE] Computed conditional probabilities for sample 1399 / 1399 [t-SNE] Mean sigma: 0.257605 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 8.9s
Quality: 0.5245831763386891 Quality: 0.5250900520090551 Quality: 0.5277482365161883 Quality: 0.5278141300047625 Quality: 0.5281812672870437 Quality: 0.5282487647186482 Quality: 0.5282835724810643
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 14.2s finished
On pattern 8
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 1100 tasks | elapsed: 2.1s [Parallel(n_jobs=10)]: Done 1350 out of 1350 | elapsed: 2.6s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 1350 samples in 0.007s... [t-SNE] Computed neighbors for 1350 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 1000 / 1350 [t-SNE] Computed conditional probabilities for sample 1350 / 1350 [t-SNE] Mean sigma: 0.269697 [t-SNE] Computed conditional probabilities in 0.090s [t-SNE] Iteration 50: error = 75.2291260, gradient norm = 0.2737437 (50 iterations in 1.059s) [t-SNE] Iteration 100: error = 76.1943512, gradient norm = 0.2520425 (50 iterations in 0.900s) [t-SNE] Iteration 150: error = 76.4783020, gradient norm = 0.2512389 (50 iterations in 0.939s) [t-SNE] Iteration 200: error = 76.2665253, gradient norm = 0.2362156 (50 iterations in 0.897s) [t-SNE] Iteration 250: error = 76.1644669, gradient norm = 0.2524489 (50 iterations in 1.004s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 76.164467 [t-SNE] Iteration 300: error = 2.3533318, gradient norm = 0.0022482 (50 iterations in 0.628s) [t-SNE] Iteration 350: error = 2.2529216, gradient norm = 0.0005477 (50 iterations in 0.592s) [t-SNE] Iteration 400: error = 2.2062101, gradient norm = 0.0002786 (50 iterations in 0.608s) [t-SNE] Iteration 450: error = 2.1835353, gradient norm = 0.0002142 (50 iterations in 0.600s) [t-SNE] Iteration 500: error = 2.1731222, gradient norm = 0.0001414 (50 iterations in 0.596s) [t-SNE] Iteration 550: error = 2.1668534, gradient norm = 0.0001421 (50 iterations in 0.612s) [t-SNE] Iteration 600: error = 2.1627951, gradient norm = 0.0000838 (50 iterations in 0.622s) [t-SNE] Iteration 650: error = 2.1594493, gradient norm = 0.0000822 (50 iterations in 0.649s) [t-SNE] Iteration 700: error = 2.1566627, gradient norm = 0.0001010 (50 iterations in 0.628s) [t-SNE] Iteration 750: error = 2.1546311, gradient norm = 0.0000798 (50 iterations in 0.634s) [t-SNE] Iteration 800: error = 2.1533625, gradient norm = 0.0000735 (50 iterations in 0.639s) [t-SNE] Iteration 850: error = 2.1520622, gradient norm = 0.0000678 (50 iterations in 0.615s) [t-SNE] Iteration 900: error = 2.1504452, gradient norm = 0.0000871 (50 iterations in 0.640s) [t-SNE] Iteration 950: error = 2.1487794, gradient norm = 0.0000941 (50 iterations in 0.629s) [t-SNE] Iteration 1000: error = 2.1468058, gradient norm = 0.0001128 (50 iterations in 0.620s) [t-SNE] KL divergence after 1000 iterations: 2.146806 [t-SNE] Computed conditional probabilities for sample 1000 / 1350 [t-SNE] Computed conditional probabilities for sample 1350 / 1350 [t-SNE] Mean sigma: 0.269697 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 9.7s
Quality: 0.4693456296488295 Quality: 0.4696248874800847 Quality: 0.47000902223586033 Quality: 0.4713582367400352 Quality: 0.47169628343720593
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 14.8s finished
On pattern 9
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 1000 tasks | elapsed: 1.3s [Parallel(n_jobs=10)]: Done 1025 out of 1025 | elapsed: 1.4s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 1025 samples in 0.006s... [t-SNE] Computed neighbors for 1025 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 1000 / 1025 [t-SNE] Computed conditional probabilities for sample 1025 / 1025 [t-SNE] Mean sigma: 0.280725 [t-SNE] Computed conditional probabilities in 0.069s [t-SNE] Iteration 50: error = 74.6192551, gradient norm = 0.2965483 (50 iterations in 0.624s) [t-SNE] Iteration 100: error = 74.7657852, gradient norm = 0.3136957 (50 iterations in 0.734s) [t-SNE] Iteration 150: error = 75.5458603, gradient norm = 0.2960848 (50 iterations in 0.649s) [t-SNE] Iteration 200: error = 77.1700439, gradient norm = 0.2908935 (50 iterations in 0.679s) [t-SNE] Iteration 250: error = 77.0156860, gradient norm = 0.2728202 (50 iterations in 0.683s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 77.015686 [t-SNE] Iteration 300: error = 2.1624596, gradient norm = 0.0029562 (50 iterations in 0.442s) [t-SNE] Iteration 350: error = 2.0693047, gradient norm = 0.0007212 (50 iterations in 0.435s) [t-SNE] Iteration 400: error = 2.0279524, gradient norm = 0.0003580 (50 iterations in 0.425s) [t-SNE] Iteration 450: error = 2.0090840, gradient norm = 0.0002520 (50 iterations in 0.433s) [t-SNE] Iteration 500: error = 1.9996345, gradient norm = 0.0001816 (50 iterations in 0.427s) [t-SNE] Iteration 550: error = 1.9934129, gradient norm = 0.0001893 (50 iterations in 0.463s) [t-SNE] Iteration 600: error = 1.9892334, gradient norm = 0.0001858 (50 iterations in 0.424s) [t-SNE] Iteration 650: error = 1.9868796, gradient norm = 0.0000963 (50 iterations in 0.414s) [t-SNE] Iteration 700: error = 1.9851108, gradient norm = 0.0000883 (50 iterations in 0.413s) [t-SNE] Iteration 750: error = 1.9839668, gradient norm = 0.0000834 (50 iterations in 0.428s) [t-SNE] Iteration 800: error = 1.9830383, gradient norm = 0.0000863 (50 iterations in 0.405s) [t-SNE] Iteration 850: error = 1.9821187, gradient norm = 0.0001108 (50 iterations in 0.436s) [t-SNE] Iteration 900: error = 1.9816219, gradient norm = 0.0000488 (50 iterations in 0.413s) [t-SNE] Iteration 950: error = 1.9810383, gradient norm = 0.0000648 (50 iterations in 0.442s) [t-SNE] Iteration 1000: error = 1.9806218, gradient norm = 0.0000608 (50 iterations in 0.417s) [t-SNE] KL divergence after 1000 iterations: 1.980622 [t-SNE] Computed conditional probabilities for sample 1000 / 1025 [t-SNE] Computed conditional probabilities for sample 1025 / 1025 [t-SNE] Mean sigma: 0.280725 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 7.6s
Quality: 0.4517417058400915 Quality: 0.45327171308981723
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 11.6s finished
On pattern 10
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 964 out of 983 | elapsed: 1.3s remaining: 0.0s [Parallel(n_jobs=10)]: Done 983 out of 983 | elapsed: 1.4s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 983 samples in 0.005s... [t-SNE] Computed neighbors for 983 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 983 / 983 [t-SNE] Mean sigma: 0.283322 [t-SNE] Computed conditional probabilities in 0.066s [t-SNE] Iteration 50: error = 73.3795929, gradient norm = 0.3287922 (50 iterations in 0.585s) [t-SNE] Iteration 100: error = 74.3241730, gradient norm = 0.3273707 (50 iterations in 0.550s) [t-SNE] Iteration 150: error = 75.0255814, gradient norm = 0.3142439 (50 iterations in 0.593s) [t-SNE] Iteration 200: error = 75.4203644, gradient norm = 0.2995750 (50 iterations in 0.566s) [t-SNE] Iteration 250: error = 76.7372971, gradient norm = 0.2876633 (50 iterations in 0.622s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 76.737297 [t-SNE] Iteration 300: error = 1.8742230, gradient norm = 0.0033371 (50 iterations in 0.445s) [t-SNE] Iteration 350: error = 1.7609910, gradient norm = 0.0006875 (50 iterations in 0.396s) [t-SNE] Iteration 400: error = 1.7192597, gradient norm = 0.0004719 (50 iterations in 0.408s) [t-SNE] Iteration 450: error = 1.7009847, gradient norm = 0.0002810 (50 iterations in 0.409s) [t-SNE] Iteration 500: error = 1.6898370, gradient norm = 0.0002556 (50 iterations in 0.415s) [t-SNE] Iteration 550: error = 1.6821525, gradient norm = 0.0003046 (50 iterations in 0.396s) [t-SNE] Iteration 600: error = 1.6729364, gradient norm = 0.0002222 (50 iterations in 0.410s) [t-SNE] Iteration 650: error = 1.6671947, gradient norm = 0.0002338 (50 iterations in 0.406s) [t-SNE] Iteration 700: error = 1.6614577, gradient norm = 0.0001968 (50 iterations in 0.403s) [t-SNE] Iteration 750: error = 1.6580011, gradient norm = 0.0001131 (50 iterations in 0.409s) [t-SNE] Iteration 800: error = 1.6559342, gradient norm = 0.0001158 (50 iterations in 0.409s) [t-SNE] Iteration 850: error = 1.6547114, gradient norm = 0.0001024 (50 iterations in 0.415s) [t-SNE] Iteration 900: error = 1.6534327, gradient norm = 0.0000822 (50 iterations in 0.401s) [t-SNE] Iteration 950: error = 1.6522349, gradient norm = 0.0001093 (50 iterations in 0.397s) [t-SNE] Iteration 1000: error = 1.6515179, gradient norm = 0.0001057 (50 iterations in 0.390s) [t-SNE] KL divergence after 1000 iterations: 1.651518 [t-SNE] Computed conditional probabilities for sample 983 / 983 [t-SNE] Mean sigma: 0.283322 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 5.7s
Quality: 0.5236966334636775 Quality: 0.5251129038305309 Quality: 0.5251905579102017 Quality: 0.5253593620533071 Quality: 0.5254513570494715 Quality: 0.5254976753479765 Quality: 0.5255217121862925 Quality: 0.5257124067354255
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 9.7s finished
On pattern 11
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 841 out of 841 | elapsed: 0.9s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 841 samples in 0.004s... [t-SNE] Computed neighbors for 841 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 841 / 841 [t-SNE] Mean sigma: 0.248166 [t-SNE] Computed conditional probabilities in 0.059s [t-SNE] Iteration 50: error = 77.2277756, gradient norm = 0.3448444 (50 iterations in 0.543s) [t-SNE] Iteration 100: error = 77.5455322, gradient norm = 0.3180839 (50 iterations in 0.532s) [t-SNE] Iteration 150: error = 78.2049561, gradient norm = 0.3302135 (50 iterations in 0.494s) [t-SNE] Iteration 200: error = 77.9611588, gradient norm = 0.3503907 (50 iterations in 0.508s) [t-SNE] Iteration 250: error = 77.2229309, gradient norm = 0.3398508 (50 iterations in 0.509s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 77.222931 [t-SNE] Iteration 300: error = 2.0021589, gradient norm = 0.0049715 (50 iterations in 0.389s) [t-SNE] Iteration 350: error = 1.9303310, gradient norm = 0.0007293 (50 iterations in 0.361s) [t-SNE] Iteration 400: error = 1.9022661, gradient norm = 0.0004107 (50 iterations in 0.368s) [t-SNE] Iteration 450: error = 1.8884239, gradient norm = 0.0002639 (50 iterations in 0.358s) [t-SNE] Iteration 500: error = 1.8799641, gradient norm = 0.0002383 (50 iterations in 0.342s) [t-SNE] Iteration 550: error = 1.8762153, gradient norm = 0.0001426 (50 iterations in 0.336s) [t-SNE] Iteration 600: error = 1.8732761, gradient norm = 0.0001479 (50 iterations in 0.340s) [t-SNE] Iteration 650: error = 1.8717470, gradient norm = 0.0000992 (50 iterations in 0.366s) [t-SNE] Iteration 700: error = 1.8687161, gradient norm = 0.0003578 (50 iterations in 0.361s) [t-SNE] Iteration 750: error = 1.8668984, gradient norm = 0.0000982 (50 iterations in 0.337s) [t-SNE] Iteration 800: error = 1.8665043, gradient norm = 0.0000811 (50 iterations in 0.328s) [t-SNE] Iteration 850: error = 1.8660126, gradient norm = 0.0000686 (50 iterations in 0.327s) [t-SNE] Iteration 900: error = 1.8653555, gradient norm = 0.0000912 (50 iterations in 0.326s) [t-SNE] Iteration 950: error = 1.8647922, gradient norm = 0.0000789 (50 iterations in 0.332s) [t-SNE] Iteration 1000: error = 1.8643903, gradient norm = 0.0000668 (50 iterations in 0.344s) [t-SNE] KL divergence after 1000 iterations: 1.864390 [t-SNE] Computed conditional probabilities for sample 841 / 841 [t-SNE] Mean sigma: 0.248166 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 5.6s
Quality: 0.4129651257245361 Quality: 0.41378601365498746 Quality: 0.4143310458235865 Quality: 0.4144059818087325 Quality: 0.41456540450597196 Quality: 0.41460327533645214
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 9.4s finished
On pattern 12
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 700 tasks | elapsed: 0.8s [Parallel(n_jobs=10)]: Done 790 out of 790 | elapsed: 0.9s finished
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 790 samples in 0.005s... [t-SNE] Computed neighbors for 790 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 790 / 790 [t-SNE] Mean sigma: 0.295889 [t-SNE] Computed conditional probabilities in 0.052s
/users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 72.8400726, gradient norm = 0.4025093 (50 iterations in 0.451s) [t-SNE] Iteration 100: error = 73.9294968, gradient norm = 0.3910526 (50 iterations in 0.428s) [t-SNE] Iteration 150: error = 75.2153778, gradient norm = 0.3777983 (50 iterations in 0.482s) [t-SNE] Iteration 200: error = 75.2524261, gradient norm = 0.3810600 (50 iterations in 0.457s) [t-SNE] Iteration 250: error = 75.4530716, gradient norm = 0.3845153 (50 iterations in 0.479s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 75.453072 [t-SNE] Iteration 300: error = 1.6123017, gradient norm = 0.0039089 (50 iterations in 0.401s) [t-SNE] Iteration 350: error = 1.5135013, gradient norm = 0.0019127 (50 iterations in 0.318s) [t-SNE] Iteration 400: error = 1.4834906, gradient norm = 0.0004699 (50 iterations in 0.302s) [t-SNE] Iteration 450: error = 1.4695507, gradient norm = 0.0002520 (50 iterations in 0.295s) [t-SNE] Iteration 500: error = 1.4634765, gradient norm = 0.0001961 (50 iterations in 0.298s) [t-SNE] Iteration 550: error = 1.4593599, gradient norm = 0.0002478 (50 iterations in 0.296s) [t-SNE] Iteration 600: error = 1.4547234, gradient norm = 0.0002287 (50 iterations in 0.298s) [t-SNE] Iteration 650: error = 1.4512810, gradient norm = 0.0002306 (50 iterations in 0.295s) [t-SNE] Iteration 700: error = 1.4488592, gradient norm = 0.0001125 (50 iterations in 0.294s) [t-SNE] Iteration 750: error = 1.4472502, gradient norm = 0.0001108 (50 iterations in 0.288s) [t-SNE] Iteration 800: error = 1.4461565, gradient norm = 0.0001458 (50 iterations in 0.300s) [t-SNE] Iteration 850: error = 1.4457376, gradient norm = 0.0000895 (50 iterations in 0.302s) [t-SNE] Iteration 900: error = 1.4450675, gradient norm = 0.0001092 (50 iterations in 0.299s) [t-SNE] Iteration 950: error = 1.4444700, gradient norm = 0.0000890 (50 iterations in 0.297s) [t-SNE] Iteration 1000: error = 1.4438299, gradient norm = 0.0001541 (50 iterations in 0.310s) [t-SNE] KL divergence after 1000 iterations: 1.443830 [t-SNE] Computed conditional probabilities for sample 790 / 790 [t-SNE] Mean sigma: 0.295889 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 4.6s
Quality: 0.5481334672227076
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 7.2s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.
On pattern 13
[Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 649 out of 668 | elapsed: 0.6s remaining: 0.0s [Parallel(n_jobs=10)]: Done 668 out of 668 | elapsed: 0.6s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 668 samples in 0.004s... [t-SNE] Computed neighbors for 668 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 668 / 668 [t-SNE] Mean sigma: 0.273794 [t-SNE] Computed conditional probabilities in 0.044s [t-SNE] Iteration 50: error = 70.9704208, gradient norm = 0.4328543 (50 iterations in 0.379s) [t-SNE] Iteration 100: error = 73.9154129, gradient norm = 0.4148211 (50 iterations in 0.384s) [t-SNE] Iteration 150: error = 74.0502319, gradient norm = 0.4109508 (50 iterations in 0.387s) [t-SNE] Iteration 200: error = 74.3150406, gradient norm = 0.3964637 (50 iterations in 0.363s) [t-SNE] Iteration 250: error = 74.0131531, gradient norm = 0.3994254 (50 iterations in 0.374s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 74.013153 [t-SNE] Iteration 300: error = 1.4616629, gradient norm = 0.0039882 (50 iterations in 0.323s) [t-SNE] Iteration 350: error = 1.3592484, gradient norm = 0.0008935 (50 iterations in 0.256s) [t-SNE] Iteration 400: error = 1.3279665, gradient norm = 0.0005045 (50 iterations in 0.263s) [t-SNE] Iteration 450: error = 1.3000320, gradient norm = 0.0007447 (50 iterations in 0.262s) [t-SNE] Iteration 500: error = 1.2867992, gradient norm = 0.0003361 (50 iterations in 0.246s) [t-SNE] Iteration 550: error = 1.2792784, gradient norm = 0.0007274 (50 iterations in 0.258s) [t-SNE] Iteration 600: error = 1.2712545, gradient norm = 0.0003257 (50 iterations in 0.259s) [t-SNE] Iteration 650: error = 1.2658114, gradient norm = 0.0003973 (50 iterations in 0.258s) [t-SNE] Iteration 700: error = 1.2606901, gradient norm = 0.0002797 (50 iterations in 0.257s) [t-SNE] Iteration 750: error = 1.2586864, gradient norm = 0.0001494 (50 iterations in 0.256s) [t-SNE] Iteration 800: error = 1.2560930, gradient norm = 0.0001448 (50 iterations in 0.259s) [t-SNE] Iteration 850: error = 1.2539028, gradient norm = 0.0001335 (50 iterations in 0.261s) [t-SNE] Iteration 900: error = 1.2531270, gradient norm = 0.0000961 (50 iterations in 0.248s) [t-SNE] Iteration 950: error = 1.2526454, gradient norm = 0.0000963 (50 iterations in 0.254s) [t-SNE] Iteration 1000: error = 1.2520977, gradient norm = 0.0000907 (50 iterations in 0.253s) [t-SNE] KL divergence after 1000 iterations: 1.252098 [t-SNE] Computed conditional probabilities for sample 668 / 668 [t-SNE] Mean sigma: 0.273794 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 4.1s
Quality: 0.543757580968836 Quality: 0.545874141268663 Quality: 0.5459031543985022 Quality: 0.5459881521864027 Quality: 0.5460974054271478
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 6.5s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers.
On pattern 14
[Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 476 out of 476 | elapsed: 0.4s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 476 samples in 0.003s... [t-SNE] Computed neighbors for 476 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 476 / 476 [t-SNE] Mean sigma: 0.315391 [t-SNE] Computed conditional probabilities in 0.032s [t-SNE] Iteration 50: error = 70.7997284, gradient norm = 0.4952686 (50 iterations in 0.230s) [t-SNE] Iteration 100: error = 73.0661545, gradient norm = 0.4838338 (50 iterations in 0.253s) [t-SNE] Iteration 150: error = 74.6547623, gradient norm = 0.4656080 (50 iterations in 0.240s) [t-SNE] Iteration 200: error = 74.0473938, gradient norm = 0.4740519 (50 iterations in 0.234s) [t-SNE] Iteration 250: error = 73.9799881, gradient norm = 0.4652984 (50 iterations in 0.248s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 73.979988 [t-SNE] Iteration 300: error = 1.3824539, gradient norm = 0.0043090 (50 iterations in 0.168s) [t-SNE] Iteration 350: error = 1.3166695, gradient norm = 0.0008499 (50 iterations in 0.180s) [t-SNE] Iteration 400: error = 1.2846380, gradient norm = 0.0008573 (50 iterations in 0.179s) [t-SNE] Iteration 450: error = 1.2741336, gradient norm = 0.0004096 (50 iterations in 0.191s) [t-SNE] Iteration 500: error = 1.2645491, gradient norm = 0.0002425 (50 iterations in 0.185s) [t-SNE] Iteration 550: error = 1.2608706, gradient norm = 0.0003635 (50 iterations in 0.197s) [t-SNE] Iteration 600: error = 1.2580866, gradient norm = 0.0001699 (50 iterations in 0.185s) [t-SNE] Iteration 650: error = 1.2564974, gradient norm = 0.0001652 (50 iterations in 0.193s) [t-SNE] Iteration 700: error = 1.2545416, gradient norm = 0.0002402 (50 iterations in 0.184s) [t-SNE] Iteration 750: error = 1.2522151, gradient norm = 0.0001601 (50 iterations in 0.186s) [t-SNE] Iteration 800: error = 1.2510250, gradient norm = 0.0001454 (50 iterations in 0.176s) [t-SNE] Iteration 850: error = 1.2468263, gradient norm = 0.0003713 (50 iterations in 0.188s) [t-SNE] Iteration 900: error = 1.2407964, gradient norm = 0.0002598 (50 iterations in 0.172s) [t-SNE] Iteration 950: error = 1.2396306, gradient norm = 0.0001171 (50 iterations in 0.195s) [t-SNE] Iteration 1000: error = 1.2392501, gradient norm = 0.0001255 (50 iterations in 0.173s) [t-SNE] KL divergence after 1000 iterations: 1.239250 [t-SNE] Computed conditional probabilities for sample 476 / 476 [t-SNE] Mean sigma: 0.315391 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 3.6s
Quality: 0.5028629126052526 Quality: 0.5042309293180464 Quality: 0.5049622100458863 Quality: 0.5053614880352861
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 5.7s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s
On pattern 15
[Parallel(n_jobs=10)]: Done 415 out of 415 | elapsed: 0.5s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 415 samples in 0.003s... [t-SNE] Computed neighbors for 415 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 415 / 415 [t-SNE] Mean sigma: 0.316621 [t-SNE] Computed conditional probabilities in 0.026s [t-SNE] Iteration 50: error = 71.9910583, gradient norm = 0.4867787 (50 iterations in 0.186s) [t-SNE] Iteration 100: error = 71.7208710, gradient norm = 0.4948164 (50 iterations in 0.198s) [t-SNE] Iteration 150: error = 72.4845963, gradient norm = 0.4877083 (50 iterations in 0.218s) [t-SNE] Iteration 200: error = 74.6255493, gradient norm = 0.4665582 (50 iterations in 0.199s) [t-SNE] Iteration 250: error = 73.0996857, gradient norm = 0.4663799 (50 iterations in 0.206s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 73.099686 [t-SNE] Iteration 300: error = 1.3495957, gradient norm = 0.0045577 (50 iterations in 0.167s) [t-SNE] Iteration 350: error = 1.2773442, gradient norm = 0.0018147 (50 iterations in 0.152s) [t-SNE] Iteration 400: error = 1.2526338, gradient norm = 0.0011467 (50 iterations in 0.148s) [t-SNE] Iteration 450: error = 1.2299687, gradient norm = 0.0011491 (50 iterations in 0.147s) [t-SNE] Iteration 500: error = 1.2112147, gradient norm = 0.0007574 (50 iterations in 0.135s) [t-SNE] Iteration 550: error = 1.2048731, gradient norm = 0.0006496 (50 iterations in 0.139s) [t-SNE] Iteration 600: error = 1.1982775, gradient norm = 0.0005839 (50 iterations in 0.141s) [t-SNE] Iteration 650: error = 1.1889912, gradient norm = 0.0004365 (50 iterations in 0.144s) [t-SNE] Iteration 700: error = 1.1833220, gradient norm = 0.0003982 (50 iterations in 0.148s) [t-SNE] Iteration 750: error = 1.1784577, gradient norm = 0.0003537 (50 iterations in 0.148s) [t-SNE] Iteration 800: error = 1.1733335, gradient norm = 0.0006216 (50 iterations in 0.145s) [t-SNE] Iteration 850: error = 1.1709014, gradient norm = 0.0003317 (50 iterations in 0.144s) [t-SNE] Iteration 900: error = 1.1693923, gradient norm = 0.0002080 (50 iterations in 0.142s) [t-SNE] Iteration 950: error = 1.1682757, gradient norm = 0.0003531 (50 iterations in 0.139s) [t-SNE] Iteration 1000: error = 1.1663661, gradient norm = 0.0001633 (50 iterations in 0.140s) [t-SNE] KL divergence after 1000 iterations: 1.166366 [t-SNE] Computed conditional probabilities for sample 415 / 415 [t-SNE] Mean sigma: 0.316621 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 3.2s
Quality: 0.4674480921671901 Quality: 0.4679962528329997 Quality: 0.469771912014876 Quality: 0.47077851454761294 Quality: 0.47081888305406544
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 5.2s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s
On pattern 16
[Parallel(n_jobs=10)]: Done 416 out of 416 | elapsed: 0.5s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 416 samples in 0.003s... [t-SNE] Computed neighbors for 416 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 416 / 416 [t-SNE] Mean sigma: 0.303102 [t-SNE] Computed conditional probabilities in 0.028s [t-SNE] Iteration 50: error = 70.9151459, gradient norm = 0.4783529 (50 iterations in 0.193s) [t-SNE] Iteration 100: error = 72.2381744, gradient norm = 0.4885525 (50 iterations in 0.213s) [t-SNE] Iteration 150: error = 72.3612137, gradient norm = 0.4698780 (50 iterations in 0.193s) [t-SNE] Iteration 200: error = 74.2076950, gradient norm = 0.4596864 (50 iterations in 0.187s) [t-SNE] Iteration 250: error = 74.5217514, gradient norm = 0.4726399 (50 iterations in 0.202s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 74.521751 [t-SNE] Iteration 300: error = 1.6584886, gradient norm = 0.0050742 (50 iterations in 0.159s) [t-SNE] Iteration 350: error = 1.5833292, gradient norm = 0.0013039 (50 iterations in 0.164s) [t-SNE] Iteration 400: error = 1.5451977, gradient norm = 0.0015070 (50 iterations in 0.153s) [t-SNE] Iteration 450: error = 1.5262612, gradient norm = 0.0004523 (50 iterations in 0.168s) [t-SNE] Iteration 500: error = 1.5088074, gradient norm = 0.0006468 (50 iterations in 0.160s) [t-SNE] Iteration 550: error = 1.4956057, gradient norm = 0.0004170 (50 iterations in 0.168s) [t-SNE] Iteration 600: error = 1.4697225, gradient norm = 0.0005171 (50 iterations in 0.156s) [t-SNE] Iteration 650: error = 1.4599879, gradient norm = 0.0005920 (50 iterations in 0.154s) [t-SNE] Iteration 700: error = 1.4475069, gradient norm = 0.0004944 (50 iterations in 0.167s) [t-SNE] Iteration 750: error = 1.4349607, gradient norm = 0.0004131 (50 iterations in 0.147s) [t-SNE] Iteration 800: error = 1.4271090, gradient norm = 0.0002935 (50 iterations in 0.144s) [t-SNE] Iteration 850: error = 1.4210329, gradient norm = 0.0002179 (50 iterations in 0.162s) [t-SNE] Iteration 900: error = 1.4126232, gradient norm = 0.0010996 (50 iterations in 0.143s) [t-SNE] Iteration 950: error = 1.4026955, gradient norm = 0.0022958 (50 iterations in 0.145s) [t-SNE] Iteration 1000: error = 1.3967738, gradient norm = 0.0002042 (50 iterations in 0.144s) [t-SNE] KL divergence after 1000 iterations: 1.396774 [t-SNE] Computed conditional probabilities for sample 416 / 416 [t-SNE] Mean sigma: 0.303102 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 3.9s
Quality: 0.4202029450603603 Quality: 0.42199831712160685
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 5.8s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s
On pattern 17 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 178 samples in 0.002s... [t-SNE] Computed neighbors for 178 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 178 / 178 [t-SNE] Mean sigma: 0.369852 [t-SNE] Computed conditional probabilities in 0.012s
[Parallel(n_jobs=10)]: Done 178 out of 178 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 63.1753845, gradient norm = 0.4897804 (50 iterations in 0.068s) [t-SNE] Iteration 100: error = 63.4707565, gradient norm = 0.5349225 (50 iterations in 0.065s) [t-SNE] Iteration 150: error = 63.8051186, gradient norm = 0.4934042 (50 iterations in 0.065s) [t-SNE] Iteration 200: error = 63.8108711, gradient norm = 0.4805364 (50 iterations in 0.066s) [t-SNE] Iteration 250: error = 62.5213013, gradient norm = 0.5299079 (50 iterations in 0.066s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 62.521301 [t-SNE] Iteration 300: error = 1.0329140, gradient norm = 0.0056447 (50 iterations in 0.059s) [t-SNE] Iteration 350: error = 0.8261480, gradient norm = 0.0038158 (50 iterations in 0.055s) [t-SNE] Iteration 400: error = 0.7880625, gradient norm = 0.0017690 (50 iterations in 0.054s) [t-SNE] Iteration 450: error = 0.7677332, gradient norm = 0.0006331 (50 iterations in 0.052s) [t-SNE] Iteration 500: error = 0.7565504, gradient norm = 0.0008320 (50 iterations in 0.053s) [t-SNE] Iteration 550: error = 0.7328743, gradient norm = 0.0004719 (50 iterations in 0.053s) [t-SNE] Iteration 600: error = 0.7328225, gradient norm = 0.0001549 (50 iterations in 0.054s) [t-SNE] Iteration 650: error = 0.7325094, gradient norm = 0.0002603 (50 iterations in 0.053s) [t-SNE] Iteration 700: error = 0.7325065, gradient norm = 0.0002457 (50 iterations in 0.052s) [t-SNE] Iteration 750: error = 0.7327114, gradient norm = 0.0002471 (50 iterations in 0.054s) [t-SNE] Iteration 800: error = 0.7321432, gradient norm = 0.0002967 (50 iterations in 0.055s) [t-SNE] Iteration 850: error = 0.7325596, gradient norm = 0.0003079 (50 iterations in 0.055s) [t-SNE] Iteration 900: error = 0.7322180, gradient norm = 0.0002893 (50 iterations in 0.054s) [t-SNE] Iteration 950: error = 0.7324260, gradient norm = 0.0002750 (50 iterations in 0.053s) [t-SNE] Iteration 1000: error = 0.7324572, gradient norm = 0.0001853 (50 iterations in 0.053s) [t-SNE] KL divergence after 1000 iterations: 0.732457 [t-SNE] Computed conditional probabilities for sample 178 / 178 [t-SNE] Mean sigma: 0.369852 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.5s
Quality: 0.4142656260469526
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.2s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s
On pattern 18 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 171 samples in 0.002s... [t-SNE] Computed neighbors for 171 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 171 / 171 [t-SNE] Mean sigma: 0.308631 [t-SNE] Computed conditional probabilities in 0.012s
[Parallel(n_jobs=10)]: Done 171 out of 171 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
[t-SNE] Iteration 50: error = 66.4836044, gradient norm = 0.4904192 (50 iterations in 0.061s) [t-SNE] Iteration 100: error = 66.3696976, gradient norm = 0.4892178 (50 iterations in 0.063s) [t-SNE] Iteration 150: error = 65.4627228, gradient norm = 0.5155222 (50 iterations in 0.072s) [t-SNE] Iteration 200: error = 63.2948875, gradient norm = 0.5188241 (50 iterations in 0.063s) [t-SNE] Iteration 250: error = 63.3056717, gradient norm = 0.5042859 (50 iterations in 0.060s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 63.305672 [t-SNE] Iteration 300: error = 1.2429020, gradient norm = 0.0052620 (50 iterations in 0.054s) [t-SNE] Iteration 350: error = 1.0786035, gradient norm = 0.0069368 (50 iterations in 0.059s) [t-SNE] Iteration 400: error = 0.9892735, gradient norm = 0.0029464 (50 iterations in 0.055s) [t-SNE] Iteration 450: error = 0.9626129, gradient norm = 0.0028574 (50 iterations in 0.059s) [t-SNE] Iteration 500: error = 0.9247011, gradient norm = 0.0063381 (50 iterations in 0.054s) [t-SNE] Iteration 550: error = 0.9132528, gradient norm = 0.0010304 (50 iterations in 0.059s) [t-SNE] Iteration 600: error = 0.9137438, gradient norm = 0.0002859 (50 iterations in 0.054s) [t-SNE] Iteration 650: error = 0.9135765, gradient norm = 0.0002681 (50 iterations in 0.055s) [t-SNE] Iteration 700: error = 0.9136220, gradient norm = 0.0000998 (50 iterations in 0.054s) [t-SNE] Iteration 750: error = 0.9137238, gradient norm = 0.0001186 (50 iterations in 0.063s) [t-SNE] Iteration 800: error = 0.9136710, gradient norm = 0.0002817 (50 iterations in 0.054s) [t-SNE] Iteration 850: error = 0.9137524, gradient norm = 0.0002846 (50 iterations in 0.054s) [t-SNE] Iteration 900: error = 0.9136436, gradient norm = 0.0001412 (50 iterations in 0.054s) [t-SNE] Iteration 900: did not make any progress during the last 300 episodes. Finished. [t-SNE] KL divergence after 900 iterations: 0.913644 [t-SNE] Computed conditional probabilities for sample 171 / 171 [t-SNE] Mean sigma: 0.308631 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 3.2s
Quality: 0.31701121898918694
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.8s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s [Parallel(n_jobs=10)]: Done 170 out of 170 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 19 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 170 samples in 0.002s... [t-SNE] Computed neighbors for 170 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 170 / 170 [t-SNE] Mean sigma: 0.374965 [t-SNE] Computed conditional probabilities in 0.012s [t-SNE] Iteration 50: error = 62.2363701, gradient norm = 0.5004377 (50 iterations in 0.059s) [t-SNE] Iteration 100: error = 61.9932404, gradient norm = 0.5175272 (50 iterations in 0.074s) [t-SNE] Iteration 150: error = 62.2714157, gradient norm = 0.5212240 (50 iterations in 0.059s) [t-SNE] Iteration 200: error = 63.2568665, gradient norm = 0.5052974 (50 iterations in 0.060s) [t-SNE] Iteration 250: error = 62.2408447, gradient norm = 0.4787686 (50 iterations in 0.066s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 62.240845 [t-SNE] Iteration 300: error = 0.9207221, gradient norm = 0.0070291 (50 iterations in 0.055s) [t-SNE] Iteration 350: error = 0.7565640, gradient norm = 0.0034082 (50 iterations in 0.052s) [t-SNE] Iteration 400: error = 0.7123841, gradient norm = 0.0019925 (50 iterations in 0.053s) [t-SNE] Iteration 450: error = 0.7003243, gradient norm = 0.0008185 (50 iterations in 0.052s) [t-SNE] Iteration 500: error = 0.6864639, gradient norm = 0.0003910 (50 iterations in 0.052s) [t-SNE] Iteration 550: error = 0.6863663, gradient norm = 0.0002857 (50 iterations in 0.064s) [t-SNE] Iteration 600: error = 0.6865433, gradient norm = 0.0003309 (50 iterations in 0.051s) [t-SNE] Iteration 650: error = 0.6864351, gradient norm = 0.0003513 (50 iterations in 0.053s) [t-SNE] Iteration 700: error = 0.6865309, gradient norm = 0.0003704 (50 iterations in 0.053s) [t-SNE] Iteration 750: error = 0.6866090, gradient norm = 0.0002646 (50 iterations in 0.053s) [t-SNE] Iteration 800: error = 0.6865823, gradient norm = 0.0002902 (50 iterations in 0.053s) [t-SNE] Iteration 850: error = 0.6866506, gradient norm = 0.0002722 (50 iterations in 0.052s) [t-SNE] Iteration 900: error = 0.6865301, gradient norm = 0.0003364 (50 iterations in 0.052s) [t-SNE] Iteration 900: did not make any progress during the last 300 episodes. Finished. [t-SNE] KL divergence after 900 iterations: 0.686530 [t-SNE] Computed conditional probabilities for sample 170 / 170 [t-SNE] Mean sigma: 0.374966 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.6s
Quality: 0.45874960384207303
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.2s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s [Parallel(n_jobs=10)]: Done 126 out of 126 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 20 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 126 samples in 0.002s... [t-SNE] Computed neighbors for 126 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 126 / 126 [t-SNE] Mean sigma: 0.360909 [t-SNE] Computed conditional probabilities in 0.009s [t-SNE] Iteration 50: error = 61.5557480, gradient norm = 0.5276205 (50 iterations in 0.042s) [t-SNE] Iteration 100: error = 60.5801239, gradient norm = 0.4247884 (50 iterations in 0.043s) [t-SNE] Iteration 150: error = 60.1556740, gradient norm = 0.5373078 (50 iterations in 0.042s) [t-SNE] Iteration 200: error = 58.6416054, gradient norm = 0.5449140 (50 iterations in 0.043s) [t-SNE] Iteration 250: error = 60.7660332, gradient norm = 0.5174773 (50 iterations in 0.043s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 60.766033 [t-SNE] Iteration 300: error = 0.8514641, gradient norm = 0.0132630 (50 iterations in 0.040s) [t-SNE] Iteration 350: error = 0.6356350, gradient norm = 0.0094601 (50 iterations in 0.039s) [t-SNE] Iteration 400: error = 0.4465708, gradient norm = 0.0054194 (50 iterations in 0.057s) [t-SNE] Iteration 450: error = 0.3862396, gradient norm = 0.0037471 (50 iterations in 0.037s) [t-SNE] Iteration 500: error = 0.3825163, gradient norm = 0.0005260 (50 iterations in 0.037s) [t-SNE] Iteration 550: error = 0.3823862, gradient norm = 0.0003410 (50 iterations in 0.038s) [t-SNE] Iteration 600: error = 0.3824446, gradient norm = 0.0006000 (50 iterations in 0.038s) [t-SNE] Iteration 650: error = 0.3823771, gradient norm = 0.0003236 (50 iterations in 0.038s) [t-SNE] Iteration 700: error = 0.3825537, gradient norm = 0.0004395 (50 iterations in 0.038s) [t-SNE] Iteration 750: error = 0.3820853, gradient norm = 0.0003868 (50 iterations in 0.038s) [t-SNE] Iteration 800: error = 0.3820627, gradient norm = 0.0002790 (50 iterations in 0.038s) [t-SNE] Iteration 850: error = 0.3820976, gradient norm = 0.0002326 (50 iterations in 0.038s) [t-SNE] Iteration 900: error = 0.3821023, gradient norm = 0.0002080 (50 iterations in 0.039s) [t-SNE] Iteration 950: error = 0.3821566, gradient norm = 0.0002988 (50 iterations in 0.038s) [t-SNE] Iteration 1000: error = 0.3823017, gradient norm = 0.0003021 (50 iterations in 0.038s) [t-SNE] KL divergence after 1000 iterations: 0.382302 [t-SNE] Computed conditional probabilities for sample 126 / 126 [t-SNE] Mean sigma: 0.360909 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.4s
Quality: 0.38599268623263744
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.6s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 129 out of 129 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 21 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 129 samples in 0.001s... [t-SNE] Computed neighbors for 129 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 129 / 129 [t-SNE] Mean sigma: 0.362306 [t-SNE] Computed conditional probabilities in 0.009s [t-SNE] Iteration 50: error = 59.3168106, gradient norm = 0.4947243 (50 iterations in 0.044s) [t-SNE] Iteration 100: error = 59.5335884, gradient norm = 0.5560592 (50 iterations in 0.044s) [t-SNE] Iteration 150: error = 63.2553139, gradient norm = 0.4746847 (50 iterations in 0.046s) [t-SNE] Iteration 200: error = 62.5930443, gradient norm = 0.5117324 (50 iterations in 0.044s) [t-SNE] Iteration 250: error = 59.8801918, gradient norm = 0.5121629 (50 iterations in 0.045s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 59.880192 [t-SNE] Iteration 300: error = 0.8318986, gradient norm = 0.0099926 (50 iterations in 0.041s) [t-SNE] Iteration 350: error = 0.4841280, gradient norm = 0.0156722 (50 iterations in 0.039s) [t-SNE] Iteration 400: error = 0.3629474, gradient norm = 0.0035076 (50 iterations in 0.038s) [t-SNE] Iteration 450: error = 0.3499402, gradient norm = 0.0019715 (50 iterations in 0.038s) [t-SNE] Iteration 500: error = 0.3476515, gradient norm = 0.0007505 (50 iterations in 0.042s) [t-SNE] Iteration 550: error = 0.3469065, gradient norm = 0.0008149 (50 iterations in 0.038s) [t-SNE] Iteration 600: error = 0.3469379, gradient norm = 0.0004027 (50 iterations in 0.038s) [t-SNE] Iteration 650: error = 0.3468359, gradient norm = 0.0004366 (50 iterations in 0.039s) [t-SNE] Iteration 700: error = 0.3467028, gradient norm = 0.0003653 (50 iterations in 0.037s) [t-SNE] Iteration 750: error = 0.3467359, gradient norm = 0.0003822 (50 iterations in 0.037s) [t-SNE] Iteration 800: error = 0.3468138, gradient norm = 0.0001589 (50 iterations in 0.037s) [t-SNE] Iteration 850: error = 0.3468759, gradient norm = 0.0004633 (50 iterations in 0.038s) [t-SNE] Iteration 900: error = 0.3467513, gradient norm = 0.0003714 (50 iterations in 0.039s) [t-SNE] Iteration 950: error = 0.3467884, gradient norm = 0.0003257 (50 iterations in 0.039s) [t-SNE] Iteration 1000: error = 0.3470546, gradient norm = 0.0004190 (50 iterations in 0.038s) [t-SNE] KL divergence after 1000 iterations: 0.347055 [t-SNE] Computed conditional probabilities for sample 129 / 129 [t-SNE] Mean sigma: 0.362306 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.4s
Quality: 0.45275358443619435
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.0s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s [Parallel(n_jobs=10)]: Done 100 out of 119 | elapsed: 0.1s remaining: 0.0s [Parallel(n_jobs=10)]: Done 119 out of 119 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 22 [t-SNE] Computing 91 nearest neighbors... [t-SNE] Indexed 119 samples in 0.002s... [t-SNE] Computed neighbors for 119 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 119 / 119 [t-SNE] Mean sigma: 0.350419 [t-SNE] Computed conditional probabilities in 0.008s [t-SNE] Iteration 50: error = 62.3915138, gradient norm = 0.4456848 (50 iterations in 0.039s) [t-SNE] Iteration 100: error = 61.2438126, gradient norm = 0.4858187 (50 iterations in 0.040s) [t-SNE] Iteration 150: error = 61.0654602, gradient norm = 0.4713223 (50 iterations in 0.039s) [t-SNE] Iteration 200: error = 62.9806786, gradient norm = 0.4832416 (50 iterations in 0.040s) [t-SNE] Iteration 250: error = 61.2592888, gradient norm = 0.5270723 (50 iterations in 0.039s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 61.259289 [t-SNE] Iteration 300: error = 1.1199603, gradient norm = 0.0060792 (50 iterations in 0.036s) [t-SNE] Iteration 350: error = 0.9785181, gradient norm = 0.0036690 (50 iterations in 0.037s) [t-SNE] Iteration 400: error = 0.8520628, gradient norm = 0.0040499 (50 iterations in 0.037s) [t-SNE] Iteration 450: error = 0.8062425, gradient norm = 0.0020114 (50 iterations in 0.037s) [t-SNE] Iteration 500: error = 0.7987872, gradient norm = 0.0009388 (50 iterations in 0.036s) [t-SNE] Iteration 550: error = 0.7951376, gradient norm = 0.0018752 (50 iterations in 0.036s) [t-SNE] Iteration 600: error = 0.7773792, gradient norm = 0.0013637 (50 iterations in 0.036s) [t-SNE] Iteration 650: error = 0.7740856, gradient norm = 0.0008910 (50 iterations in 0.037s) [t-SNE] Iteration 700: error = 0.7731956, gradient norm = 0.0005275 (50 iterations in 0.037s) [t-SNE] Iteration 750: error = 0.7741313, gradient norm = 0.0005709 (50 iterations in 0.037s) [t-SNE] Iteration 800: error = 0.7735215, gradient norm = 0.0004709 (50 iterations in 0.036s) [t-SNE] Iteration 850: error = 0.7735570, gradient norm = 0.0004123 (50 iterations in 0.037s) [t-SNE] Iteration 900: error = 0.7736790, gradient norm = 0.0004041 (50 iterations in 0.037s) [t-SNE] Iteration 950: error = 0.7734330, gradient norm = 0.0003342 (50 iterations in 0.037s) [t-SNE] Iteration 1000: error = 0.7733749, gradient norm = 0.0003284 (50 iterations in 0.036s) [t-SNE] KL divergence after 1000 iterations: 0.773375 [t-SNE] Computed conditional probabilities for sample 119 / 119 [t-SNE] Mean sigma: 0.350419 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.5s
Quality: 0.2877562951139348 Quality: 0.28814500379972163 Quality: 0.2889116607258183 Quality: 0.289946016955707
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.0s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s [Parallel(n_jobs=10)]: Done 83 out of 83 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 23 [t-SNE] Computing 82 nearest neighbors... [t-SNE] Indexed 83 samples in 0.001s... [t-SNE] Computed neighbors for 83 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 83 / 83 [t-SNE] Mean sigma: 0.400290 [t-SNE] Computed conditional probabilities in 0.006s [t-SNE] Iteration 50: error = 60.2459030, gradient norm = 0.5059912 (50 iterations in 0.031s) [t-SNE] Iteration 100: error = 61.6908302, gradient norm = 0.4967867 (50 iterations in 0.027s) [t-SNE] Iteration 150: error = 52.3510323, gradient norm = 0.5833091 (50 iterations in 0.027s) [t-SNE] Iteration 200: error = 60.0622177, gradient norm = 0.5044972 (50 iterations in 0.026s) [t-SNE] Iteration 250: error = 55.1840248, gradient norm = 0.5776298 (50 iterations in 0.027s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 55.184025 [t-SNE] Iteration 300: error = 0.9933779, gradient norm = 0.0036852 (50 iterations in 0.026s) [t-SNE] Iteration 350: error = 0.7818521, gradient norm = 0.0018329 (50 iterations in 0.025s) [t-SNE] Iteration 400: error = 0.7241877, gradient norm = 0.0007034 (50 iterations in 0.024s) [t-SNE] Iteration 450: error = 0.6987470, gradient norm = 0.0006036 (50 iterations in 0.024s) [t-SNE] Iteration 500: error = 0.6781743, gradient norm = 0.0011136 (50 iterations in 0.024s) [t-SNE] Iteration 550: error = 0.6584630, gradient norm = 0.0003404 (50 iterations in 0.024s) [t-SNE] Iteration 600: error = 0.6438742, gradient norm = 0.0005362 (50 iterations in 0.024s) [t-SNE] Iteration 650: error = 0.6344414, gradient norm = 0.0005979 (50 iterations in 0.024s) [t-SNE] Iteration 700: error = 0.5633917, gradient norm = 0.0049757 (50 iterations in 0.029s) [t-SNE] Iteration 750: error = 0.4505624, gradient norm = 0.0088744 (50 iterations in 0.025s) [t-SNE] Iteration 800: error = 0.3916780, gradient norm = 0.0094958 (50 iterations in 0.024s) [t-SNE] Iteration 850: error = 0.3482358, gradient norm = 0.0090957 (50 iterations in 0.025s) [t-SNE] Iteration 900: error = 0.3461141, gradient norm = 0.0026536 (50 iterations in 0.025s) [t-SNE] Iteration 950: error = 0.3460120, gradient norm = 0.0040578 (50 iterations in 0.024s) [t-SNE] Iteration 1000: error = 0.3459589, gradient norm = 0.0020413 (50 iterations in 0.024s) [t-SNE] KL divergence after 1000 iterations: 0.345959 [t-SNE] Computed conditional probabilities for sample 83 / 83 [t-SNE] Mean sigma: 0.400290 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.9s
Quality: 0.27028018430103773 Quality: 0.2723676394507729 Quality: 0.27267256325406486
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.4s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.1s [Parallel(n_jobs=10)]: Done 88 out of 88 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 24 [t-SNE] Computing 87 nearest neighbors... [t-SNE] Indexed 88 samples in 0.001s... [t-SNE] Computed neighbors for 88 samples in 0.001s... [t-SNE] Computed conditional probabilities for sample 88 / 88 [t-SNE] Mean sigma: 0.372425 [t-SNE] Computed conditional probabilities in 0.006s [t-SNE] Iteration 50: error = 56.7020378, gradient norm = 0.5326113 (50 iterations in 0.028s) [t-SNE] Iteration 100: error = 59.7857437, gradient norm = 0.4686510 (50 iterations in 0.033s) [t-SNE] Iteration 150: error = 60.5366745, gradient norm = 0.5407948 (50 iterations in 0.029s) [t-SNE] Iteration 200: error = 55.3903313, gradient norm = 0.5886902 (50 iterations in 0.028s) [t-SNE] Iteration 250: error = 60.6383133, gradient norm = 0.5147340 (50 iterations in 0.028s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 60.638313 [t-SNE] Iteration 300: error = 1.0429138, gradient norm = 0.0201459 (50 iterations in 0.027s) [t-SNE] Iteration 350: error = 0.9445135, gradient norm = 0.0012919 (50 iterations in 0.029s) [t-SNE] Iteration 400: error = 0.8124121, gradient norm = 0.0010718 (50 iterations in 0.028s) [t-SNE] Iteration 450: error = 0.7586132, gradient norm = 0.0006607 (50 iterations in 0.031s) [t-SNE] Iteration 500: error = 0.7273122, gradient norm = 0.0006110 (50 iterations in 0.028s) [t-SNE] Iteration 550: error = 0.6777688, gradient norm = 0.0012032 (50 iterations in 0.028s) [t-SNE] Iteration 600: error = 0.5921614, gradient norm = 0.0010545 (50 iterations in 0.027s) [t-SNE] Iteration 650: error = 0.5459732, gradient norm = 0.0007089 (50 iterations in 0.027s) [t-SNE] Iteration 700: error = 0.5309885, gradient norm = 0.0002930 (50 iterations in 0.027s) [t-SNE] Iteration 750: error = 0.5233821, gradient norm = 0.0002497 (50 iterations in 0.026s) [t-SNE] Iteration 800: error = 0.5223851, gradient norm = 0.0002026 (50 iterations in 0.026s) [t-SNE] Iteration 850: error = 0.5210365, gradient norm = 0.0002305 (50 iterations in 0.026s) [t-SNE] Iteration 900: error = 0.5094959, gradient norm = 0.0005367 (50 iterations in 0.027s) [t-SNE] Iteration 950: error = 0.6173713, gradient norm = 0.0016093 (50 iterations in 0.029s) [t-SNE] Iteration 1000: error = 0.6076700, gradient norm = 0.0008887 (50 iterations in 0.029s) [t-SNE] KL divergence after 1000 iterations: 0.607670 [t-SNE] Computed conditional probabilities for sample 88 / 88 [t-SNE] Mean sigma: 0.372425 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.3s
Quality: 0.2635070006898537
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 3.9s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 40 tasks | elapsed: 0.0s [Parallel(n_jobs=10)]: Done 83 out of 83 | elapsed: 0.1s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 25 [t-SNE] Computing 82 nearest neighbors... [t-SNE] Indexed 83 samples in 0.001s... [t-SNE] Computed neighbors for 83 samples in 0.000s... [t-SNE] Computed conditional probabilities for sample 83 / 83 [t-SNE] Mean sigma: 0.410106 [t-SNE] Computed conditional probabilities in 0.006s [t-SNE] Iteration 50: error = 55.7762756, gradient norm = 0.4524075 (50 iterations in 0.025s) [t-SNE] Iteration 100: error = 56.6495438, gradient norm = 0.4862151 (50 iterations in 0.026s) [t-SNE] Iteration 150: error = 55.1362419, gradient norm = 0.5132678 (50 iterations in 0.026s) [t-SNE] Iteration 200: error = 57.3279991, gradient norm = 0.4688133 (50 iterations in 0.027s) [t-SNE] Iteration 250: error = 55.3338966, gradient norm = 0.5250687 (50 iterations in 0.027s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 55.333897 [t-SNE] Iteration 300: error = 0.8681784, gradient norm = 0.0048146 (50 iterations in 0.025s) [t-SNE] Iteration 350: error = 0.6730463, gradient norm = 0.0028069 (50 iterations in 0.025s) [t-SNE] Iteration 400: error = 0.4938832, gradient norm = 0.0018726 (50 iterations in 0.025s) [t-SNE] Iteration 450: error = 0.5486009, gradient norm = 0.0012860 (50 iterations in 0.031s) [t-SNE] Iteration 500: error = 0.4640044, gradient norm = 0.0005018 (50 iterations in 0.027s) [t-SNE] Iteration 550: error = 0.4305297, gradient norm = 0.0006401 (50 iterations in 0.027s) [t-SNE] Iteration 600: error = 0.3998616, gradient norm = 0.0002505 (50 iterations in 0.027s) [t-SNE] Iteration 650: error = 0.3391790, gradient norm = 0.0005763 (50 iterations in 0.026s) [t-SNE] Iteration 700: error = 0.3340296, gradient norm = 0.0000772 (50 iterations in 0.025s) [t-SNE] Iteration 750: error = 0.3319021, gradient norm = 0.0000970 (50 iterations in 0.025s) [t-SNE] Iteration 800: error = 0.3260489, gradient norm = 0.0001614 (50 iterations in 0.025s) [t-SNE] Iteration 850: error = 0.3159459, gradient norm = 0.0004266 (50 iterations in 0.025s) [t-SNE] Iteration 900: error = 0.2969579, gradient norm = 0.0001645 (50 iterations in 0.025s) [t-SNE] Iteration 950: error = 0.2949832, gradient norm = 0.0000522 (50 iterations in 0.024s) [t-SNE] Iteration 1000: error = 0.2948467, gradient norm = 0.0000861 (50 iterations in 0.025s) [t-SNE] KL divergence after 1000 iterations: 0.294847 [t-SNE] Computed conditional probabilities for sample 83 / 83 [t-SNE] Mean sigma: 0.410106 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.3s
Quality: 0.3937480757457587 Quality: 0.39422276955319757
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 3.8s finished [Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 66 out of 66 | elapsed: 0.0s finished /users/avanti/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_base.py:168: EfficiencyWarning: Precomputed sparse input was not sorted by data. EfficiencyWarning)
On pattern 26 [t-SNE] Computing 65 nearest neighbors... [t-SNE] Indexed 66 samples in 0.001s... [t-SNE] Computed neighbors for 66 samples in 0.000s... [t-SNE] Computed conditional probabilities for sample 66 / 66 [t-SNE] Mean sigma: 0.427425 [t-SNE] Computed conditional probabilities in 0.004s [t-SNE] Iteration 50: error = 56.6754265, gradient norm = 0.4592462 (50 iterations in 0.021s) [t-SNE] Iteration 100: error = 56.5149765, gradient norm = 0.5495380 (50 iterations in 0.021s) [t-SNE] Iteration 150: error = 59.3411064, gradient norm = 0.4945654 (50 iterations in 0.022s) [t-SNE] Iteration 200: error = 57.9221916, gradient norm = 0.4821824 (50 iterations in 0.021s) [t-SNE] Iteration 250: error = 52.7093811, gradient norm = 0.5930980 (50 iterations in 0.021s) [t-SNE] KL divergence after 250 iterations with early exaggeration: 52.709381 [t-SNE] Iteration 300: error = 1.0188861, gradient norm = 0.0023486 (50 iterations in 0.021s) [t-SNE] Iteration 350: error = 0.8252379, gradient norm = 0.0009243 (50 iterations in 0.021s) [t-SNE] Iteration 400: error = 0.7776315, gradient norm = 0.0006667 (50 iterations in 0.020s) [t-SNE] Iteration 450: error = 0.7034658, gradient norm = 0.0012419 (50 iterations in 0.020s) [t-SNE] Iteration 500: error = 0.6638021, gradient norm = 0.0004397 (50 iterations in 0.020s) [t-SNE] Iteration 550: error = 0.6357763, gradient norm = 0.0006059 (50 iterations in 0.020s) [t-SNE] Iteration 600: error = 0.5992208, gradient norm = 0.0005499 (50 iterations in 0.020s) [t-SNE] Iteration 650: error = 0.5183435, gradient norm = 0.0017800 (50 iterations in 0.020s) [t-SNE] Iteration 700: error = 0.4722012, gradient norm = 0.0004888 (50 iterations in 0.020s) [t-SNE] Iteration 750: error = 0.4378884, gradient norm = 0.0012679 (50 iterations in 0.021s) [t-SNE] Iteration 800: error = 0.4301499, gradient norm = 0.0002857 (50 iterations in 0.021s) [t-SNE] Iteration 850: error = 0.4241584, gradient norm = 0.0024828 (50 iterations in 0.021s) [t-SNE] Iteration 900: error = 0.4182353, gradient norm = 0.0002393 (50 iterations in 0.030s) [t-SNE] Iteration 950: error = 0.4080231, gradient norm = 0.0004536 (50 iterations in 0.021s) [t-SNE] Iteration 1000: error = 0.4186725, gradient norm = 0.0045672 (50 iterations in 0.021s) [t-SNE] KL divergence after 1000 iterations: 0.418672 [t-SNE] Computed conditional probabilities for sample 66 / 66 [t-SNE] Mean sigma: 0.427425 Beginning preprocessing + Leiden
[Parallel(n_jobs=10)]: Using backend LokyBackend with 10 concurrent workers. [Parallel(n_jobs=10)]: Done 30 tasks | elapsed: 2.9s
Quality: 0.22417160537808511
[Parallel(n_jobs=10)]: Done 50 out of 50 | elapsed: 4.4s finished
In [7]:
import h5py
import modisco.util
reload(modisco.util)
import os
file_path = "v0.5.13.0.hdf5"
if (os.path.exists(file_path)):
os.remove(file_path)
grp = h5py.File(file_path, "w")
results.save_hdf5(grp)
grp.close()
In [8]:
from modisco.visualization import viz_sequence
%matplotlib inline
hdf5_results = h5py.File(file_path,"r")
metacluster_names = [
x.decode("utf-8") for x in
list(hdf5_results["metaclustering_results"]
["all_metacluster_names"][:])]
all_patterns = []
background = np.array([0.27, 0.23, 0.23, 0.27])
for metacluster_name in metacluster_names:
print(metacluster_name)
metacluster_grp = (hdf5_results["metacluster_idx_to_submetacluster_results"]
[metacluster_name])
print("activity pattern:",metacluster_grp["activity_pattern"][:])
all_pattern_names = [x.decode("utf-8") for x in
list(metacluster_grp["seqlets_to_patterns_result"]
["patterns"]["all_pattern_names"][:])]
if (len(all_pattern_names)==0):
print("No motifs found for this activity pattern")
for pattern_name in all_pattern_names:
print(metacluster_name, pattern_name)
all_patterns.append((metacluster_name, pattern_name))
pattern = metacluster_grp["seqlets_to_patterns_result"]["patterns"][pattern_name]
print("total seqlets:",len(pattern["seqlets_and_alnmts"]["seqlets"]))
print("Task 0 hypothetical scores:")
viz_sequence.plot_weights(pattern["Nanog_profile_wn_hypothetical_contribs"]["fwd"])
print("Task 0 actual importance scores:")
viz_sequence.plot_weights(pattern["Nanog_profile_wn_contrib_scores"]["fwd"])
print("onehot, fwd and rev:")
viz_sequence.plot_weights(np.array(pattern["sequence"]["fwd"]))
viz_sequence.plot_weights(np.array(pattern["sequence"]["rev"]))
hdf5_results.close()
metacluster_0 activity pattern: [1] metacluster_0 pattern_0 total seqlets: 10449 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_1 total seqlets: 7117 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_2 total seqlets: 3051 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_3 total seqlets: 2581 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_4 total seqlets: 2353 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_5 total seqlets: 2172 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_6 total seqlets: 1665 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_7 total seqlets: 1399 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_8 total seqlets: 1350 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_9 total seqlets: 1025 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_10 total seqlets: 983 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_11 total seqlets: 841 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_12 total seqlets: 790 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_13 total seqlets: 668 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_14 total seqlets: 476 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_15 total seqlets: 415 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_16 total seqlets: 416 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_17 total seqlets: 178 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_18 total seqlets: 171 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_19 total seqlets: 170 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_20 total seqlets: 126 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_21 total seqlets: 129 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_22 total seqlets: 119 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_23 total seqlets: 83 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_24 total seqlets: 88 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_25 total seqlets: 83 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
metacluster_0 pattern_26 total seqlets: 66 Task 0 hypothetical scores:
Task 0 actual importance scores:
onehot, fwd and rev:
In [ ]: