YASS -- Yet Another Scheduler Simulator

Objectives:

  • Take a set of jobs (htc or hpc), and simulate their execution on a fixed set of compute jobs.
  • For scheduling, use First-Come-Frist-Serve (FCFS) algorithm with backfilling (no fair-sharing and gang-scheduling; similar to SLURM's scheduling with sched/backfill configuration).
  • Record enough information through the execution, to evaluate different preemption policies with respect to the amount of wasted cycles.

Tunable parameters:

  • Mode: hpc, htc, or peregrine.
  • Number of nodes in the cluster (set when Cluster object is created).
  • Number of nodes that will be preempted (set when Cluster object is created; default: 50% of cluster nodes).
  • Experiment duration -- run execution until all jobs complete or this time limited is exceeded (passed as a parameter in Evaluation.run()).

Imports and contol variables

In [13]:
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys, os, random
import seaborn as sns
from collections import OrderedDict
import random
import operator
from copy import deepcopy

# Define the mode of operation:
# htc - run single-node jobs
# hpc - run multi-node jobs
# peregrine - run jobs from Peregrine system
MODE = "htc"

# Number of times jobs from the selected workload are repeated (and shuffled in the final list)
JOB_REPEAT_COUNT = 10

# Standard figure size
FSIZE = (10,4)

# Perform error checking 
if not MODE in set(["hpc", "htc", "peregrine"]):
    print "Error: specified incorrect mode of operation"
    sys.exit(1)

Run simulation (this might take a long time)

  • During simulation, progress is reported in the form: "simulated_time_in_seconds..." at moments when one of the jobs finishes.
  • Results of the simulation are saved to: wasted_cycles.csv, preemptions.csv, and job_execution.csv.
In [3]:
jobs_dir = "jobs_" + MODE + "/"

# Iterate through each jobs, index - Order
#for index, row in jobs.iterrows():
#    print index, row["Runtime"], row["NodeCount"]

class Workload(object):
    def __init__(self, jobs_dir):
        self.jobs_dir = jobs_dir
        self.jobs = pd.read_csv(jobs_dir + "summary.csv")
        
        if MODE == "htc":
            #print self.jobs.head()
            #print self.jobs.tail()
            pass
        
        if MODE == "hpc":
            # For comparison only use the specified number of jobs
            # self.jobs = self.jobs.iloc[:200]
            pass
        
        if MODE == "peregrine":
            total_job_count = len(self.jobs)
            self.jobs = self.jobs.loc[(self.jobs["NodeCount"] <= 20) & (self.jobs["Runtime"] <= 24*3600.0) & \
                                      (self.jobs["Runtime"] >= 0.01)]
            print "Selected Peregrine jobs: %d out of %d" % (len(self.jobs), total_job_count)
        
        if JOB_REPEAT_COUNT > 1:
            print "Original job count: ", len(self.jobs)
            # Add shuffled copies of jobs to make the workload longer
            all_jobs = pd.DataFrame()
            for i in range(JOB_REPEAT_COUNT):
                new_jobs = self.jobs.iloc[np.random.permutation(len(self.jobs))]
                all_jobs = all_jobs.append(new_jobs)
            all_jobs.reset_index(drop=True, inplace=True)
            all_jobs['Order'] = all_jobs.index
            self.jobs = all_jobs
            print "New job count: ", len(self.jobs)
        
        # Print job stats
        print self.jobs.describe()
        total_node_hours = 0.0
        for idx, j in self.jobs.iterrows():
            total_node_hours += j["NodeCount"] * j ["Runtime"] / 3600.0
        print "Total node-hours in the selected workload: %f" % total_node_hours
        
        self.jobs = self.jobs.set_index("Order")
        self.jobs["NodeCount"] = self.jobs["NodeCount"].astype(int)
        self.jobs["State"] = "P"
        self.jobs["Started"] = ""
        self.jobs["Completed"] = ""

class Cluster(object):
    
    def __init__(self, node_count, preempt_count=None):
        """
        self.nodes -- dict where keys are node names and values are elapsed time for the running job or -1 for idle
                        (if not -1, always growing to corresponding job total runtims)
        self.running_jobs -- dict where keys are jobs IDs and the values are dicts:
            "Runtime": xxx - total runtime for this job (fixed for each job)
            "Elapsed": xxx - time it has been running so far (growing up to "Runtime")
            "NodeCount": xxx - number of nodes used by this jobs
            "Nodes": [xxx,yyy,zzz] - list of names of the nodes used by this job
        """
        self.node_count = node_count
        self.nodes = OrderedDict([ ("node-%d" % id, -1) for id in range(self.node_count) ])
        self.idle_node_set = set([ "node-%d" % id for id in range(self.node_count)]) 
        self.running_jobs = OrderedDict()
        if not preempt_count:
            self.preempt_count = int(self.node_count / 2.0)
        
    def place_job(self, job_id, job_runtime, job_node_count):
        """
        Simulate scheduling and executing a job
        If a job with given attributes can be executed: update node structures and return true
        Otherwise, return false
        """
        if len(self.idle_node_set) == 0:
            return False
        candidates = list(self.idle_node_set)
        if len(candidates) < job_node_count:
            return False
        else:
            # print "Node count", job_node_count
            selected_nodes = candidates[:int(job_node_count)]
            # print "Job %d started on nodes: %s" % (job_id, str(selected_nodes))
            for node in selected_nodes:
                self.nodes[node] = 0.0  
                self.idle_node_set.remove(node)
                self.running_jobs[job_id] = {"Runtime": job_runtime, "Elapsed": 0.0, \
                                             "NodeCount": job_node_count, "Nodes": selected_nodes}
            return True
    
    def fast_forward(self, starting_time, show_progress=True):
        """
        Simulate running all jobs until the first one finishes.
        Returns time to the moment when the next job will finish and the list of IDs of the finished job,
        subtract that time from runtimes for 
        all jobs and nodes, and free up the nodes for the finished jobs
        """
        # Find min remaining time across all jobs
        time_step = min([job["Runtime"] - job["Elapsed"] for id, job in self.running_jobs.items()])
        # print "Moving forward by %d seconds from state of running jobs %s" % (time_step, self.running_jobs) 
        # print "Moving forward by %d seconds" % (time_step) 
        if time_step < 0:
            print "Error: moving back in time!"
            sys.exit(1)
        
        # within time_step interval no job will finish; simulate policy evaluation with currently running jobs
        PolicyEval.evaluate(self.preempt_count, self.nodes, self.running_jobs, starting_time, time_step)
        
        finished_jobs = []
        for id, job in self.running_jobs.items():
            job["Elapsed"] += time_step
            if abs(job["Runtime"] - job["Elapsed"]) < 0.001:
                # Consider finished 
                for node in job["Nodes"]:
                    self.nodes[node] = -1
                    self.idle_node_set.add(node)
                del self.running_jobs[id]
                # print "Job %d completed on nodes: %s" % (id, str(job["Nodes"]))
                finished_jobs.append(id)
            else:
                for node in job["Nodes"]:
                    self.nodes[node] += time_step
        
        if show_progress:
            sys.stdout.write("%d..." % starting_time)
            sys.stdout.flush()
        return time_step, finished_jobs  

class PolicyEval(object):
    # Class/static dataframe
    wasted_cycles = pd.DataFrame()
    preemptions = pd.DataFrame()
    grace_periods = [60,120,1200,1800]
    eval_counter = 0
    
    @staticmethod
    def evaluate(preemt_count, nodes, running_jobs, starting_time, duration, time_delta=30):

        # Evaluate every second
        for dt in range(0, int(duration), time_delta):

            # Simulate the state of nodes and jobs at curr_time + dt 
            
            # Nodes: if idle, leave idle; otherwise, fast forward by dt
            future_nodes =  OrderedDict([ (node, -1)  if elap == -1 else (node, elap + dt) \
                                                  for node, elap in nodes.items()]) 
            # Jobs: fast forward all jobs by dt
            future_running_jobs = deepcopy(running_jobs)
            for id, job in future_running_jobs.items():
                job["Elapsed"] += dt
            
            RANDOM_vec, RANDOM_nodes = PolicyEval.RANDOM(preemt_count, future_nodes, future_running_jobs)
            RANDOM_vec.update({"Policy": "RANDOM", "Time": starting_time + dt})
            
            # Simple, original version: estimate LIFO and FIFO separately
            #LIFO_vec, LIFO_nodes = PolicyEval.LIFO(preemt_count, future_nodes, future_running_jobs)
            #LIFO_vec.update({"Policy": "LIFO", "Time": starting_time + dt})
            #FIFO_vec, FIFO_nodes = PolicyEval.FIFO(preemt_count, future_nodes, future_running_jobs)
            #FIFO_vec.update({"Policy": "FIFO", "Time": starting_time + dt})
            # Optimized version: estimate LIFO and FIFO together and save some cycles by avoiding repeated sorts
            LIFO_vec, LIFO_nodes, FIFO_vec, FIFO_nodes = PolicyEval.LIFO_and_FIFO(preemt_count, future_nodes, future_running_jobs)
            LIFO_vec.update({"Policy": "LIFO", "Time": starting_time + dt})
            FIFO_vec.update({"Policy": "FIFO", "Time": starting_time + dt})
            
            PAP_vec, PAP_nodes = PolicyEval.PAP(preemt_count, future_nodes, future_running_jobs)
            PAP_vec.update({"Policy": "PAP", "Time": starting_time + dt})
            
            PolicyEval.preemptions = PolicyEval.preemptions.append(pd.DataFrame([RANDOM_vec, LIFO_vec, FIFO_vec, PAP_vec])) 
                
            for gp in PolicyEval.grace_periods:
                RANDOM_wc = PolicyEval.get_wasted_cycles(future_running_jobs, RANDOM_nodes, gp) 
                LIFO_wc = PolicyEval.get_wasted_cycles(future_running_jobs, LIFO_nodes, gp) 
                FIFO_wc = PolicyEval.get_wasted_cycles(future_running_jobs, FIFO_nodes, gp) 
                PAP_wc = PolicyEval.get_wasted_cycles(future_running_jobs, PAP_nodes, gp) 
        
                PolicyEval.wasted_cycles = PolicyEval.wasted_cycles.append(pd.DataFrame([ {"Policy": "RANDOM", "GracePeriod": gp ,"Time": starting_time + dt, "WastedCycles": RANDOM_wc},
                                                                                         {"Policy": "LIFO", "GracePeriod": gp, "Time": starting_time + dt, "WastedCycles": LIFO_wc},
                                                                                         {"Policy": "FIFO", "GracePeriod": gp, "Time": starting_time + dt, "WastedCycles": FIFO_wc},
                                                                                         {"Policy": "PAP", "GracePeriod": gp, "Time": starting_time + dt, "WastedCycles": PAP_wc} ])) 
        PolicyEval.eval_counter += 1
        # Periodically save eval results 
        if PolicyEval.eval_counter % 100 == 0 and PolicyEval.eval_counter > 0:
            PolicyEval.save()
            
        
    @staticmethod
    def RANDOM(preemt_count, nodes, running_jobs):
        """ 
        Given running jobs and node states, decide on what nodes to kill according to the RANDOM policy.
        Random policy igores the runtimes.
        
        Returned:
        - p_vec -- preemption_vector (dict) where keys are nodes and values are preemption values in [0,1]
        - p_nodes -- preempted_nodes, list with preemt_count nodes that should be preempted
        """
        
        p_vec = OrderedDict([(node,random.uniform(0, 1)) for node in nodes.keys()])
        
        # Proper version: sort by value, get keys, and only select the necessary number
        # p_nodes = [pair[0] for pair in sorted(p_vec.items(), key=operator.itemgetter(1))][:preemt_count] 
        
        # Optimized version: ignore assigned values, return random sample
        p_nodes = random.sample(p_vec.keys(), preemt_count)
        return p_vec, p_nodes   
    
    @staticmethod
    def LIFO(preemt_count, nodes, running_jobs):
        """ 
        Given a row from monitor data, decide on what nodes to kill according to the LIFO policy.
        LIFO, last-in-first-out policy, creates preemption vectors based on current job runtimes:
        the longer a job has been running, the higher the value of the node it is running on is.
        
        Deletion is simulated by ranking the available nodes according to this policy,
        and selecting the specified number of nodes at the beginning of the list (sorted in the value-ascending order).
        Preemption vectors takes form of a dict: {<node>:<value>}
        List of deleted nodes is returned.
        
        Returned:
        - p_vec -- preemption_vector (dict) where keys are nodes and values are preemption values in [0,1]
        - p_nodes -- preempted_nodes, list with preemt_count nodes that should be preempted
        """
        tmp = deepcopy(nodes)        
        for node in tmp.keys():
            if tmp[node] == -1:
                tmp[node] = 0
        # Sort by value
        tmp = sorted(tmp.items(), key=operator.itemgetter(1))
        # Scale -- divide by the largest value (if not zero)
        max_elapsed = tmp[-1][1]
        if abs(max_elapsed) > 0.0001:
            p_vec = OrderedDict([ (node,elap/max_elapsed) for node,elap in tmp])
        else:
            p_vec = OrderedDict(tmp)
        
        p_nodes = p_vec.keys()[:preemt_count] 
        return p_vec, p_nodes 
    
    @staticmethod
    def FIFO(preemt_count, nodes, running_jobs):
        """ 
        Given a row from monitor data, decide on what nodes to kill according to the FIFO policy.
        FIFO, first-in-first-out policy, creates preemption vectors based on current job runtimes:
        the longer a job has been running, the lower the value of the node it is running on is.
        
        Deletion is simulated by ranking the available nodes according to this policy,
        and selecting the specified number of nodes at the beginning of the list (sorted in the value-descending order).
        Preemption vectors takes form of a dict: {<node>:<value>}
        List of deleted nodes is returned.
        
        Returned:
        - p_vec -- preemption_vector (dict) where keys are nodes and values are preemption values in [0,1]
        - p_nodes -- preempted_nodes, list with preemt_count nodes that should be preempted
        """
        tmp = deepcopy(nodes)
        for node in tmp.keys():
            if tmp[node] == -1:
                tmp[node] = 0
        # Sort by value
        tmp = sorted(tmp.items(), key=operator.itemgetter(1))
        # Scale -- divide by the largest value (if not zero)
        max_elapsed = tmp[-1][1]
        if abs(max_elapsed) > 0.0001:
            p_vec = OrderedDict([ (node,1.0-elap/max_elapsed) for node,elap in tmp ])
        else:
            # all values are zeros
            p_vec = OrderedDict([ (node,1.0) for node,elap in tmp ])
        
        # last preemt_count elements in the list
        p_nodes = p_vec.keys()[-preemt_count:] 
        return p_vec, p_nodes
    
    @staticmethod
    def LIFO_and_FIFO(preemt_count, nodes, running_jobs):
        """
        Optimized version that combines two previous functions
        """
        # This part is the same for both LIFO and FIFO
        tmp = dict([ (node,0.0) if elap == -1 else (node,elap) for node, elap in nodes.items() ]) 
        # Sort by value
        tmp = sorted(tmp.items(), key=operator.itemgetter(1))
        # Scale -- divide by the largest value (if not zero)
        max_elapsed = tmp[-1][1]
        
        # Policy-specific operations
        if abs(max_elapsed) > 0.0001:
            LIFO_vec = OrderedDict([ (node,elap/max_elapsed) for node,elap in tmp])
            FIFO_vec = OrderedDict([ (node,1.0-elap/max_elapsed) for node,elap in tmp])
        else:
            LIFO_vec = dict(tmp)
            FIFO_vec = {node:1.0 for node,elap in tmp}
        
        LIFO_nodes = LIFO_vec.keys()[:preemt_count] 
        FIFO_nodes = FIFO_vec.keys()[-preemt_count:] 
        return LIFO_vec, LIFO_nodes, FIFO_vec, FIFO_nodes 
    
    @staticmethod
    def PAP(preemt_count, nodes, running_jobs):
        """ 
        Given a row from monitor data, decide on what nodes to kill according to the PAP policy.
        PAP, Parallel-Aware Preemption policy, creates preemption vectors based on current job runtimes --
        the longer a job has been running, the lower the value of the node it is running on is --
        and the number of nodes used by each job.
        
        Deletion is simulated by ranking the available nodes according to this policy,
        and selecting the specified number of nodes at the beginning of the list (sorted in the value-descending order).
        Preemption vectors takes form of a dict: {<node>:<value>}
        List of deleted nodes is returned.
        
        Returned:
        - p_vec -- preemption_vector (dict) where keys are nodes and values are preemption values in [0,1]
        - p_nodes -- preempted_nodes, list with preemt_count nodes that should be preempted
        """
        # Include idle nodes 
        tmp = {node:0.0 for node, elap in nodes.items() if elap == -1}
        # Include loaded nodes
        for id, job in running_jobs.items():
            total_elapsed = job["NodeCount"] * job["Elapsed"]
            for node in job["Nodes"]:
                tmp[node] = total_elapsed
        
        tmp = sorted(tmp.items(), key=operator.itemgetter(1))
        # Scale -- divide by the largest value (if not zero)
        max_elapsed = tmp[-1][1]
        if abs(max_elapsed) > 0.0001:
            p_vec = OrderedDict([ (node,elap/max_elapsed) for node,elap in tmp])
        else:
            p_vec = OrderedDict(tmp)
        
        p_nodes = p_vec.keys()[:preemt_count] 
        return p_vec, p_nodes 
     
    @staticmethod    
    def get_wasted_cycles(running_jobs, preemted_nodes, grace_period):
        total = 0
        p_set = set(preemted_nodes)
        for id, job in running_jobs.items():
            # If find this intersection, the entire job will be preempted
            if len(set(job["Nodes"]) & p_set):
                if grace_period < job['Runtime'] - job['Elapsed']:
                    # currently running job won't finish before the end of grace period
                    total += job["NodeCount"] * (job["Elapsed"] + grace_period)
        return total
    
    @staticmethod
    def save():
        PolicyEval.wasted_cycles.to_csv("wasted_cycles.csv")
        PolicyEval.preemptions.to_csv("preemptions.csv")

class Execution(object):
    
    def __init__(self, cluster, workload):
        self.cluster = cluster
        self.workload = workload
        self.timer = 0
        self.timer_when_no_pending_jobs = 0
        self.is_timer_when_no_pending_jobs_set = False
        
    def run(self, terminate_at_timer=None):
        while True: 
            #print "Time: %f" % self.timer
            if terminate_at_timer and self.timer >= terminate_at_timer:
                print "Stopping because specified time exceeded"
                break
            
            pending_jobs = self.workload.jobs[self.workload.jobs["State"]=="P"]
            if (len(pending_jobs) == 0) and (not self.is_timer_when_no_pending_jobs_set):
                self.timer_when_no_pending_jobs = self.timer
                self.is_timer_when_no_pending_jobs_set = True
                
            # print "Number of pending jobs: %d" % len(pending_jobs) 
            
            if len(self.workload.jobs[self.workload.jobs["State"] != "C"]) == 0:
                print "All jobs completed. Finish execution"
                break
                
            placed_any = False
            # Scheduling phase
            for id, job in pending_jobs.iterrows():
                placed = self.cluster.place_job(id, job["Runtime"], job["NodeCount"])       
                if placed:
                    placed_any = True
                    self.workload.jobs.set_value(id, 'State', "R")
                    self.workload.jobs.set_value(id, 'Started', self.timer)
                if len(self.cluster.idle_node_set) == 0:
                    # Can't schedule any more jobs
                    break
            
            # Time-stepping phase
            if self.cluster.running_jobs:
                time_step, finished_jobs = self.cluster.fast_forward(starting_time=self.timer) 
                self.timer += time_step
                for id in finished_jobs:
                    self.workload.jobs.set_value(id, 'State', "C")
                    self.workload.jobs.set_value(id, 'Completed', self.timer)
        
        # End of execution
        PolicyEval.save()
                    
w = Workload(jobs_dir)
c = Cluster(20)
x = Execution(c, w)
x.run()
w.jobs.to_csv('job_execution.csv')

print "Final value of timer: %f seconds (%f hours)" % (x.timer, x.timer/3600.0)
print "Time when number of pending job is zero: %f seconds (%f hours)" % (x.timer_when_no_pending_jobs, x.timer_when_no_pending_jobs/3600.0)
Original job count:  1120
New job count:  11200
              Order  NodeCount       Runtime
count  11200.000000    11200.0  11200.000000
mean    5599.500000        1.0    443.265179
std     3233.305842        0.0   1015.780504
min        0.000000        1.0     54.000000
25%     2799.750000        1.0     84.000000
50%     5599.500000        1.0    106.000000
75%     8399.250000        1.0    134.250000
max    11199.000000        1.0   7140.000000
Total node-hours in the selected workload: 1379.047222
0...66...73...77...80...81...88...91...97...103...127...148...157...166...176...178...189...192...213...214...244...260...261...266...272...289...299...300...318...329...338...346...359...368...386...388...400...450...452...456...463...469...499...515...520...537...547...555...557...563...604...612...615...636...645...667...699...706...721...736...742...820...822...846...879...890...899...904...925...1007...1008...1013...1030...1111...1117...1143...1179...1225...1253...1345...1367...1455...1463...1539...1659...1691...1781...1903...1983...2056...2144...2370...2514...2595...2630...2660...2691...2699...2788...2792...2793...2803...2831...2838...2862...2864...2888...2890...2902...2906...2953...2968...2991...2995...3021...3028...3033...3056...3101...3104...3112...3114...3124...3158...3160...3194...3207...3209...3215...3230...3237...3243...3248...3265...3267...3273...3284...3318...3321...3323...3335...3336...3365...3367...3368...3395...3407...3431...3445...3470...3476...3504...3507...3523...3528...3533...3550...3570...3577...3586...3619...3631...3637...3645...3656...3675...3679...3683...3684...3705...3708...3742...3745...3780...3785...3801...3809...3813...3852...3853...3857...3906...3915...3921...3927...3943...3976...3978...4001...4002...4019...4051...4065...4069...4079...4080...4108...4134...4143...4152...4159...4178...4201...4205...4245...4266...4275...4290...4312...4337...4369...4400...4409...4416...4456...4503...4510...4515...4537...4545...4587...4594...4633...4640...4649...4682...4740...4754...4764...4813...4833...4839...4868...4886...4926...4953...5014...5028...5051...5083...5098...5099...5137...5145...5179...5186...5212...5218...5227...5296...5318...5338...5354...5416...5421...5426...5442...5495...5519...5532...5544...5568...5594...5611...5627...5646...5686...5688...5714...5743...5769...5771...5805...5820...5850...5867...5877...5890...5901...5950...5968...5973...5992...6033...6044...6049...6101...6137...6139...6141...6165...6200...6229...6253...6265...6269...6302...6320...6350...6354...6372...6389...6401...6412...6418...6441...6449...6515...6531...6542...6577...6592...6599...6649...6661...6675...6679...6712...6731...6751...6783...6793...6804...6818...6831...6880...6897...6909...6918...6946...7016...7030...7043...7089...7094...7136...7156...7220...7230...7269...7314...7371...7388...7399...7441...7462...7474...7485...7503...7519...7525...7541...7548...7559...7583...7606...7607...7624...7635...7691...7693...7700...7731...7732...7741...7745...7769...7809...7821...7840...7842...7859...7874...7943...7954...7960...8029...8044...8057...8078...8081...8088...8092...8103...8114...8135...8177...8178...8192...8198...8209...8211...8216...8289...8295...8300...8305...8311...8313...8339...8369...8385...8424...8433...8447...8459...8472...8481...8503...8521...8547...8561...8574...8582...8595...8624...8663...8669...8688...8698...8731...8746...8750...8764...8773...8784...8792...8811...8819...8825...8843...8865...8873...8888...8916...8928...8939...8963...9005...9030...9038...9040...9050...9060...9073...9080...9086...9130...9131...9141...9157...9167...9173...9200...9219...9221...9249...9263...9265...9276...9295...9315...9345...9358...9371...9373...9396...9415...9445...9451...9462...9472...9496...9534...9540...9548...9561...9568...9617...9619...9622...9623...9675...9738...9746...9752...9753...9768...9796...9829...9852...9853...9870...9876...9896...9936...9956...9970...9987...10036...10037...10047...10065...10138...10148...10154...10192...10206...10219...10232...10252...10288...10290...10314...10332...10337...10352...10384...10429...10432...10456...10469...10511...10529...10550...10560...10571...10628...10642...10643...10691...10697...10700...10724...10766...10800...10810...10832...10873...10889...10899...10920...10962...10971...11023...11051...11070...11144...11150...11171...11219...11263...11294...11296...11333...11389...11398...11467...11475...11547...11550...11610...11622...11716...11723...11797...11833...11908...11930...11942...12009...12020...12076...12083...12095...12168...12186...12209...12265...12282...12290...12348...12357...12392...12397...12433...12445...12448...12452...12473...12512...12529...12534...12642...12684...12740...12808...12849...12886...12915...12959...12984...12987...13033...13046...13060...13065...13124...13135...13139...13252...13270...13294...13319...13334...13370...13385...13404...13411...13419...13476...13479...13480...13515...13516...13536...13568...13576...13603...13639...13648...13654...13668...13678...13687...13706...13723...13746...13753...13769...13807...13863...13882...13897...13912...13915...13930...13937...13979...14004...14017...14025...14050...14054...14094...14098...14124...14129...14150...14175...14179...14191...14205...14241...14271...14273...14278...14353...14371...14388...14411...14443...14489...14511...14574...14593...14605...14651...14667...14684...14756...14781...14828...14874...14877...14940...14980...15014...15023...15090...15093...15106...15159...15160...15188...15241...15258...15310...15316...15340...15390...15410...15421...15468...15490...15507...15580...15614...15660...15679...15773...15898...16014...16180...16213...16299...16301...16357...16379...16398...16430...16467...16529...16565...16632...16646...16659...16699...16727...16736...16744...16784...16814...16833...16839...16841...16858...16863...16907...16910...16934...16947...16985...16993...17005...17046...17071...17074...17083...17090...17106...17143...17165...17187...17190...17191...17210...17222...17267...17289...17290...17302...17320...17322...17337...17344...17373...17378...17396...17428...17439...17452...17455...17461...17478...17486...17526...17539...17545...17556...17586...17591...17603...17666...17685...17686...17692...17694...17708...17731...17761...17798...17809...17812...17823...17834...17848...17894...17916...17923...17959...17960...18005...18013...18028...18044...18050...18079...18114...18121...18135...18154...18186...18209...18230...18250...18271...18297...18298...18307...18335...18343...18383...18393...18447...18462...18464...18477...18515...18547...18560...18570...18610...18644...18665...18693...18721...18734...18735...18778...18829...18835...18867...18878...18901...18925...18951...19006...19015...19016...19029...19031...19041...19090...19100...19106...19134...19136...19185...19211...19219...19229...19268...19320...19328...19354...19372...19398...19451...19462...19481...19526...19572...19578...19612...19643...19658...19703...19713...19725...19765...19794...19826...19836...19872...19878...19908...19929...19949...19968...20004...20005...20027...20056...20113...20158...20171...20197...20232...20250...20297...20315...20321...20328...20329...20334...20389...20431...20433...20439...20471...20542...20553...20567...20575...20587...20658...20659...20678...20680...20710...20746...20778...20823...20824...20833...20843...20876...20905...20957...20988...20993...21001...21057...21075...21111...21112...21139...21153...21156...21165...21216...21231...21236...21240...21252...21258...21315...21336...21346...21364...21385...21402...21417...21420...21425...21433...21451...21489...21502...21506...21546...21568...21573...21598...21630...21634...21642...21662...21701...21705...21709...21710...21713...21765...21785...21797...21810...21844...21853...21875...21885...21907...21933...21956...21959...21984...22005...22023...22066...22080...22095...22098...22121...22137...22157...22172...22185...22188...22216...22235...22244...22252...22330...22341...22348...22370...22377...22383...22413...22423...22456...22461...22497...22504...22507...22519...22539...22590...22596...22610...22616...22617...22620...22631...22668...22678...22692...22698...22699...22704...22713...22726...22754...22774...22776...22814...22830...22843...22845...22862...22876...22893...22925...22928...22942...22982...22998...23004...23008...23027...23029...23049...23073...23080...23083...23086...23091...23094...23106...23111...23125...23169...23175...23198...23202...23205...23206...23223...23250...23259...23283...23284...23292...23314...23317...23337...23351...23353...23367...23402...23407...23408...23428...23452...23462...23464...23484...23492...23519...23525...23526...23550...23560...23571...23574...23594...23611...23678...23681...23684...23687...23691...23701...23712...23763...23771...23772...23782...23785...23816...23847...23860...23878...23886...23887...23907...23920...23942...23957...23968...23980...23991...23994...24035...24052...24063...24083...24093...24102...24135...24136...24143...24165...24180...24203...24207...24208...24239...24274...24291...24297...24298...24311...24316...24325...24346...24384...24395...24401...24406...24440...24459...24463...24464...24492...24514...24521...24528...24545...24553...24568...24603...24606...24618...24628...24661...24667...24699...24710...24737...24743...24769...24783...24788...24812...24855...24858...24888...24897...24902...24934...24984...24998...25014...25020...25080...25102...25108...25130...25153...25180...25199...25258...25278...25297...25298...25356...25361...25413...25442...25535...25536...25621...25651...25735...25788...25862...25891...25944...25962...26025...26108...26123...26191...26192...26284...26338...26373...26446...26452...26526...26530...26624...26725...26734...26735...26782...26809...26851...26852...26859...26862...26882...26942...26955...26962...26977...26984...27047...27048...27056...27119...27124...27178...27184...27198...27205...27257...27274...27326...27332...27341...27352...27362...27381...27418...27438...27468...27474...27527...27532...27553...27578...27605...27616...27620...27671...27682...27698...27702...27705...27766...27771...27772...27783...27791...27796...27841...27857...27863...27865...27878...27885...27896...27923...27938...27957...27974...27976...27988...28048...28060...28071...28099...28106...28128...28158...28167...28178...28198...28241...28297...28309...28312...28322...28376...28461...28464...28757...28885...28903...28979...28991...29041...29059...29069...29108...29135...29187...29205...29213...29246...29255...29273...29323...29329...29356...29388...29396...29410...29463...29471...29475...29537...29539...29552...29581...29614...29649...29676...29694...29716...29739...29776...29789...29798...29804...29885...29895...29900...29963...29975...30033...30036...30046...30053...30079...30108...30132...30146...30162...30179...30206...30218...30220...30226...30281...30294...30303...30311...30323...30350...30378...30379...30422...30438...30441...30459...30501...30543...30562...30603...30631...30666...30684...30697...30711...30774...30788...30793...30816...30838...30858...30872...30892...30923...30947...30951...30956...30957...30959...30976...30991...30998...31018...31042...31045...31046...31047...31050...31060...31089...31095...31096...31117...31120...31130...31145...31158...31161...31169...31176...31202...31210...31258...31260...31268...31297...31322...31326...31370...31372...31404...31413...31421...31444...31470...31483...31498...31524...31539...31551...31590...31631...31665...31670...31696...31732...31738...31739...31745...31776...31818...31827...31831...31859...31896...31950...31953...31961...31993...32012...32029...32061...32073...32077...32115...32125...32157...32162...32193...32199...32207...32209...32228...32231...32252...32267...32306...32308...32313...32333...32342...32381...32395...32402...32430...32436...32439...32447...32448...32475...32516...32527...32532...32533...32544...32554...32575...32577...32606...32611...32630...32635...32636...32652...32663...32669...32675...32703...32727...32744...32746...32749...32768...32770...32780...32797...32819...32850...32857...32863...32884...32911...32950...32959...32971...32989...32995...33004...33038...33042...33053...33093...33100...33102...33115...33129...33165...33182...33191...33199...33224...33271...33278...33280...33282...33296...33306...33364...33399...33411...33413...33417...33446...33506...33513...33533...33602...33636...33685...33717...33753...33809...33829...33841...33887...33888...33925...33946...33958...33985...34018...34034...34054...34065...34079...34082...34124...34139...34150...34192...34213...34230...34273...34302...34330...34364...34398...34432...34485...34534...34580...34604...34640...34659...34729...34747...34759...34820...34846...34876...34884...34966...34970...35002...35011...35070...35083...35098...35114...35145...35192...35257...35266...35273...35275...35349...35371...35407...35409...35484...35486...35489...35519...35549...35554...35602...35629...35636...35654...35704...35732...35765...35813...35818...35826...35875...35893...35895...35947...35948...35950...35958...35978...36001...36005...36022...36049...36052...36064...36090...36091...36106...36108...36127...36129...36133...36136...36160...36164...36173...36183...36199...36204...36205...36209...36233...36237...36245...36264...36269...36270...36274...36315...36317...36328...36331...36338...36356...36369...36382...36398...36404...36418...36420...36426...36438...36456...36458...36459...36480...36515...36523...36539...36546...36550...36578...36581...36599...36605...36635...36670...36671...36673...36690...36695...36712...36739...36768...36778...36785...36786...36799...36815...36857...36860...36869...36871...36903...36921...36930...36948...36952...36995...37023...37053...37060...37062...37074...37109...37111...37135...37136...37152...37211...37216...37228...37250...37265...37323...37333...37352...37428...37435...37449...37530...37533...37540...37614...37620...37701...37723...37742...37744...37780...37838...37862...37875...37909...37939...37964...37973...37974...38002...38009...38080...38084...38106...38154...38174...38183...38201...38227...38229...38247...38282...38318...38320...38321...38405...38463...38469...38545...38559...38639...38707...38735...38804...38810...38879...38890...38902...38918...38928...38977...38994...39004...39015...39061...39067...39075...39128...39148...39170...39198...39228...39245...39262...39321...39334...39373...39435...39453...39455...39477...39485...39532...39544...39562...39563...39619...39622...39649...39665...39675...39692...39752...39763...39765...39767...39778...39780...39831...39839...39843...39848...39884...39887...39913...39914...39948...39973...39994...40020...40042...40051...40112...40128...40133...40149...40197...40209...40224...40246...40265...40269...40277...40313...40330...40345...40356...40363...40366...40411...40417...40428...40447...40471...40487...40493...40510...40529...40548...40551...40592...40593...40595...40608...40632...40691...40702...40724...40727...40799...40814...40816...40819...40892...40912...40946...40968...40979...40991...40997...41052...41055...41112...41117...41146...41162...41241...41271...41273...41319...41370...41384...41387...41442...41460...41483...41531...41590...41598...41608...41674...41686...41736...41752...41822...41833...41853...41861...41912...41936...41950...41957...42048...42050...42089...42134...42161...42170...42225...42264...42266...42333...42337...42362...42372...42460...42480...42510...42519...42536...42565...42587...42593...42602...42630...42651...42679...42681...42685...42710...42737...42744...42757...42799...42823...42827...42835...42857...42895...42922...42935...42953...42975...42989...43004...43008...43042...43085...43094...43116...43129...43161...43185...43211...43216...43243...43259...43282...43296...43318...43331...43359...43361...43386...43412...43415...43450...43474...43498...43525...43551...43571...43587...43619...43632...43660...43693...43717...43723...43778...43790...43844...43871...43918...43927...43949...43950...43953...44025...44028...44032...44042...44118...44139...44218...44231...44293...44363...44388...44405...44411...44490...44517...44569...44609...44624...44690...44705...44723...44753...44796...44817...44823...44837...44911...44919...44935...44999...45003...45036...45068...45083...45160...45315...45345...45400...45424...45471...45508...45583...45595...45643...45672...45680...45689...45727...45741...45750...45784...45816...45873...45876...45895...45957...45969...45994...46026...46027...46040...46049...46073...46105...46109...46126...46128...46131...46156...46197...46209...46225...46230...46250...46251...46259...46281...46299...46311...46316...46321...46371...46381...46395...46405...46414...46416...46427...46453...46459...46466...46476...46487...46488...46492...46523...46560...46576...46577...46579...46593...46604...46612...46630...46643...46655...46673...46697...46728...46733...46740...46747...46766...46768...46809...46826...46827...46837...46856...46872...46896...46905...46922...46947...46951...46954...46973...46993...47019...47047...47053...47058...47077...47096...47138...47139...47149...47154...47178...47218...47223...47237...47258...47350...47361...47370...47405...47493...47516...47610...47723...47805...47853...47874...47949...47957...47964...48008...48029...48053...48088...48097...48099...48168...48197...48203...48254...48298...48343...48376...48412...48418...48431...48482...48490...48498...48527...48559...48566...48623...48647...48672...48702...48717...48757...48777...48813...48838...48866...48888...48910...48915...48946...48959...48994...49011...49056...49081...49098...49112...49117...49131...49143...49165...49190...49222...49226...49246...49254...49279...49288...49290...49321...49333...49345...49350...49412...49416...49431...49440...49449...49510...49514...49552...49570...49579...49588...49604...49654...49655...49657...49662...49677...49732...49781...49782...49804...49839...49855...49869...49870...49910...49951...49958...49959...49968...49987...50028...50037...50049...50065...50069...50087...50102...50116...50166...50167...50171...50209...50212...50222...50251...50263...50292...50312...50316...50339...50346...50354...50391...50424...50448...50450...50458...50470...50477...50482...50498...50528...50542...50550...50558...50569...50584...50619...50625...50640...50647...50655...50666...50693...50718...50765...50770...50841...50858...50862...50933...50979...50986...50988...51046...51058...51096...51109...51163...51203...51232...51245...51268...51270...51312...51316...51331...51389...51398...51435...51479...51536...51548...51597...51602...51609...51662...51666...51674...51682...51720...51786...51788...51792...51822...51858...51868...51884...51901...51947...51948...51957...52041...52057...52074...52092...52110...52154...52158...52174...52185...52222...52247...52249...52258...52307...52308...52309...52364...52370...52387...52390...52418...52447...52452...52454...52469...52471...52489...52520...52532...52543...52564...52579...52597...52637...52640...52661...52676...52688...52720...52742...52753...52798...52840...52870...52872...52910...52942...52978...52984...53024...53029...53075...53083...53110...53157...53158...53168...53189...53203...53207...53255...53259...53281...53292...53300...53305...53338...53369...53381...53409...53413...53450...53462...53473...53490...53501...53511...53553...53555...53566...53584...53598...53618...53620...53623...53634...53693...53695...53705...53708...53737...53751...53764...53769...53786...53806...53821...53844...53883...53909...53919...53942...53963...54014...54027...54030...54056...54096...54119...54134...54184...54193...54214...54248...54271...54276...54300...54334...54335...54369...54383...54397...54426...54435...54452...54504...54509...54515...54516...54535...54583...54584...54625...54629...54638...54663...54679...54701...54720...54742...54743...54754...54797...54801...54808...54818...54832...54858...54878...54895...54901...54905...54928...54953...54970...54975...54986...55011...55023...55026...55044...55055...55096...55112...55131...55133...55140...55145...55181...55182...55188...55215...55228...55235...55273...55275...55294...55308...55323...55326...55349...55367...55391...55410...55421...55438...55461...55492...55547...55551...55562...55569...55581...55609...55652...55654...55699...55708...55721...55781...55784...55831...55855...55895...55917...55939...55983...55993...56013...56075...56104...56123...56151...56165...56190...56241...56251...56293...56303...56315...56349...56380...56440...56452...56458...56482...56527...56561...56567...56581...56620...56632...56652...56692...56714...56739...56767...56836...56845...56931...56969...57039...57172...57242...57327...57354...57406...57417...57434...57436...57508...57512...57548...57595...57623...57633...57634...57684...57731...57751...57774...57776...57796...57822...57892...57894...57909...57934...57968...57971...57993...58006...58021...58029...58062...58074...58087...58096...58103...58106...58133...58166...58172...58181...58209...58233...58241...58245...58275...58301...58305...58319...58322...58335...58341...58357...58384...58412...58431...58435...58447...58453...58455...58470...58530...58536...58556...58562...58566...58573...58582...58598...58612...58637...58642...58650...58668...58671...58690...58693...58712...58746...58749...58775...58786...58804...58821...58843...58881...58891...58957...58973...58992...59013...59032...59056...59111...59116...59118...59169...59197...59204...59255...59272...59292...59304...59336...59349...59371...59407...59436...59444...59451...59492...59526...59544...59555...59559...59614...59657...59664...59717...59746...59753...59795...59803...59871...59877...59904...59918...59944...60010...60011...60016...60090...60100...60113...60213...60214...60300...60301...60331...60486...60568...60710...60772...60867...60897...60977...61000...61005...61088...61112...61144...61158...61172...61226...61245...61249...61259...61279...61315...61337...61340...61360...61383...61396...61397...61424...61443...61461...61463...61468...61473...61486...61525...61546...61557...61558...61559...61577...61587...61622...61638...61655...61685...61691...61712...61725...61747...61757...61760...61770...61830...61854...61858...61867...61879...61928...61948...61974...61979...62025...62052...62054...62114...62143...62159...62171...62204...62207...62226...62234...62289...62325...62344...62351...62376...62398...62445...62460...62476...62489...62492...62519...62528...62565...62570...62603...62616...62640...62643...62693...62702...62714...62731...62732...62775...62798...62823...62855...62858...62888...62915...62920...62938...62947...62997...63015...63056...63080...63150...63158...63159...63192...63197...63220...63299...63327...63328...63333...63335...63390...63423...63431...63451...63468...63512...63522...63528...63585...63635...63674...63733...63754...63774...63817...63854...63885...63953...63995...63998...64019...64108...64129...64157...64179...64214...64226...64239...64274...64286...64289...64326...64330...64344...64351...64365...64379...64433...64445...64453...64485...64522...64534...64540...64554...64566...64631...64636...64644...64682...64697...64709...64740...64757...64772...64830...64837...64845...64854...64877...64927...64933...64942...64946...64955...64974...64987...65016...65042...65057...65089...65097...65154...65163...65186...65208...65222...65240...65241...65257...65306...65313...65320...65321...65386...65389...65399...65409...65426...65428...65485...65499...65510...65529...65533...65539...65574...65575...65593...65595...65609...65616...65633...65672...65682...65690...65691...65700...65710...65730...65746...65765...65793...65810...65827...65833...65854...65873...65903...65932...65940...65948...65976...65997...66023...66058...66068...66070...66082...66096...66173...66187...66208...66209...66225...66243...66247...66271...66292...66311...66329...66348...66386...66390...66419...66420...66465...66499...66511...66561...66579...66598...66602...66614...66689...66698...66707...66719...66735...66757...66790...66828...66829...66839...66904...66921...66961...66973...66976...66984...67046...67059...67076...67078...67108...67135...67141...67154...67186...67200...67205...67223...67231...67248...67290...67345...67355...67364...67419...67426...67444...67451...67484...67502...67561...67571...67589...67680...67696...67756...67798...67841...67885...67916...67945...68009...68040...68079...68100...68128...68147...68175...68192...68230...68241...68261...68291...68326...68396...68413...68443...68533...68542...68557...68563...68624...68628...68670...68684...68721...68745...68782...68787...68825...68887...68912...68913...68921...68927...68964...68981...68990...69003...69054...69058...69071...69075...69127...69188...69194...69213...69257...69272...69321...69328...69331...69342...69412...69425...69435...69436...69469...69520...69563...69564...69584...69602...69610...69635...69656...69662...69679...69699...69729...69736...69738...69776...69777...69817...69822...69825...69844...69873...69891...69913...69939...69952...69954...69961...69969...70020...70029...70038...70039...70090...70105...70113...70116...70121...70136...70145...70172...70192...70195...70215...70226...70249...70263...70270...70285...70287...70300...70317...70348...70351...70356...70357...70368...70383...70401...70402...70429...70430...70439...70461...70467...70494...70500...70504...70520...70547...70571...70588...70611...70634...70644...70648...70662...70698...70746...70761...70778...70780...70813...70820...70835...70859...70895...70902...70912...70924...70949...70991...70993...70999...71010...71065...71077...71089...71123...71127...71128...71170...71205...71221...71251...71261...71277...71335...71445...71448...71533...71562...71631...71658...71688...71737...71773...71806...71846...71905...71917...71939...71980...72014...72018...72074...72127...72170...72238...72240...72286...72315...72341...72418...72488...72572...72581...72682...72698...72771...72821...72843...72885...72899...72944...72987...73074...73110...73119...73173...73211...73214...73217...73264...73292...73303...73309...73330...73369...73370...73373...73397...73410...73454...73463...73474...73476...73480...73564...73569...73578...73601...73608...73658...73659...73664...73691...73692...73706...73728...73731...73774...73788...73817...73820...73835...73837...73884...73901...73920...73930...73950...74004...74011...74015...74033...74045...74056...74086...74118...74119...74136...74156...74167...74206...74237...74271...74278...74325...74336...74351...74392...74403...74452...74460...74467...74472...74537...74545...74562...74569...74580...74583...74632...74648...74658...74673...74694...74701...74718...74728...74730...74748...74801...74806...74808...74825...74852...74883...74886...74927...74940...74949...74964...74975...75001...75002...75042...75044...75081...75082...75099...75123...75135...75156...75183...75193...75203...75210...75225...75263...75308...75324...75354...75361...75389...75399...75412...75423...75505...75509...75520...75547...75566...75629...75633...75635...75643...75713...75723...75773...75803...75847...75851...75862...75891...75940...75941...75969...76021...76050...76066...76095...76124...76170...76189...76212...76254...76268...76286...76320...76338...76376...76389...76408...76434...76463...76466...76493...76515...76539...76544...76555...76600...76612...76626...76627...76649...76661...76679...76685...76703...76722...76726...76738...76781...76788...76790...76804...76813...76820...76823...76827...76839...76872...76884...76892...76897...76907...76910...76922...76940...76950...76970...76972...76974...76983...76990...76993...76998...77007...77026...77027...77061...77065...77068...77093...77097...77115...77120...77127...77137...77170...77171...77216...77221...77227...77240...77264...77298...77309...77341...77379...77382...77398...77419...77458...77470...77490...77534...77541...77554...77593...77609...77633...77641...77658...77704...77725...77744...77753...77806...77823...77840...77846...77872...77898...77937...77976...77985...78005...78067...78147...78196...78230...78285...78305...78374...78398...78460...78536...78563...78633...78731...78755...78809...78877...78910...78922...78957...79003...79049...79118...79192...79215...79221...79295...79386...79426...79468...79475...79553...79556...79594...79643...79669...79692...79703...79737...79746...79772...79787...79808...79815...79889...79892...79907...79925...79930...79948...79959...80006...80008...80024...80025...80032...80039...80064...80093...80105...80120...80156...80197...80199...80210...80238...80246...80259...80299...80300...80305...80341...80386...80387...80388...80417...80422...80443...80474...80504...80514...80533...80538...80542...80544...80553...80568...80593...80612...80639...80678...80686...80688...80705...80708...80737...80793...80829...80861...80865...80897...80940...80995...81001...81032...81049...81115...81123...81158...81180...81181...81257...81399...81469...81496...81576...81609...81682...81735...81788...81809...81857...81877...81891...81929...81937...81957...81969...82008...82060...82083...82115...82156...82218...82222...82285...82351...82359...82431...82466...82539...82573...82605...82656...82675...82720...82757...82763...82799...82817...82829...82833...82850...82899...82901...82932...82970...82999...83007...83045...83076...83095...83126...83148...83149...83171...83175...83217...83226...83232...83247...83254...83275...83281...83307...83310...83335...83350...83414...83416...83439...83454...83458...83501...83520...83532...83537...83541...83544...83552...83554...83582...83598...83615...83623...83633...83679...83682...83706...83717...83724...83772...83775...83776...83797...83798...83810...83821...83834...83849...83862...83909...83925...83931...83936...83947...83953...83955...83984...84035...84043...84063...84066...84069...84077...84108...84114...84134...84135...84162...84185...84190...84214...84218...84247...84268...84278...84300...84302...84317...84321...84332...84375...84376...84382...84398...84404...84409...84451...84452...84473...84486...84488...84499...84527...84540...84565...84593...84605...84606...84621...84625...84664...84686...84707...84713...84730...84733...84749...84762...84765...84783...84786...84792...84803...84816...84864...84872...84873...84879...84899...84914...84935...84949...84967...84972...84992...85024...85025...85066...85083...85146...85157...85197...85246...85260...85273...85314...85359...85369...85376...85393...85454...85489...85498...85513...85567...85574...85615...85617...85650...85661...85691...85747...85772...85803...85832...85858...85861...85927...85930...85954...85978...86005...86021...86060...86072...86086...86135...86152...86200...86212...86214...86216...86247...86283...86314...86321...86329...86343...86353...86362...86398...86407...86430...86446...86474...86479...86525...86539...86544...86556...86566...86591...86653...86673...86678...86688...86694...86702...86715...86772...86774...86783...86794...86823...86842...86882...86899...86900...86916...86963...86984...86993...86997...87014...87075...87087...87096...87098...87110...87116...87171...87172...87205...87241...87332...87352...87400...87442...87508...87512...87574...87600...87657...87705...87743...87757...87793...87824...87839...87847...87885...87921...88014...88054...88064...88066...88108...88113...88134...88206...88209...88242...88312...88377...88386...88412...88475...88505...88528...88605...88606...88634...88636...88701...88707...88728...88734...88797...88816...88821...88929...88934...89029...89032...89125...89141...89160...89208...89224...89237...89242...89283...89290...89309...89347...89351...89364...89369...89397...89403...89407...89433...89450...89466...89473...89482...89533...89536...89544...89617...89622...89632...89689...89702...89705...89745...89780...89800...89802...89810...89820...89864...89877...89898...89901...89908...89936...89958...89979...89982...90031...90038...90051...90068...90094...90105...90128...90147...90188...90230...90244...90266...90303...90309...90312...90340...90342...90382...90388...90400...90410...90420...90424...90443...90473...90501...90506...90523...90533...90541...90548...90556...90607...90608...90636...90647...90658...90667...90673...90676...90715...90739...90740...90749...90778...90785...90799...90816...90831...90847...90853...90856...90893...90901...90902...90914...90917...90919...90935...90943...90947...90981...91004...91036...91044...91047...91048...91058...91086...91108...91112...91114...91125...91134...91142...91147...91150...91163...91165...91191...91211...91215...91238...91242...91245...91277...91283...91292...91299...91308...91313...91322...91360...91380...91381...91395...91432...91433...91438...91465...91506...91517...91521...91552...91616...91627...91630...91633...91709...91721...91748...91797...91838...91883...92018...92080...92088...92196...92213...92214...92281...92317...92359...92457...92490...92550...92622...92833...92841...92938...93061...93108...93190...93220...93249...93333...93336...93423...93490...93561...93680...93688...93774...93792...93881...93899...93917...93950...94001...94050...94088...94107...94144...94170...94172...94186...94233...94256...94261...94295...94304...94313...94333...94334...94341...94345...94363...94364...94397...94400...94422...94425...94439...94441...94463...94495...94507...94528...94545...94552...94563...94584...94586...94605...94628...94652...94667...94673...94693...94735...94742...94743...94767...94782...94813...94816...94848...94867...94873...94930...94944...94989...94990...94996...95055...95072...95084...95087...95097...95106...95161...95164...95166...95171...95190...95220...95229...95252...95260...95264...95268...95276...95278...95301...95320...95365...95371...95373...95384...95412...95432...95440...95447...95476...95481...95494...95507...95521...95524...95536...95541...95564...95599...95600...95605...95655...95658...95671...95679...95683...95729...95732...95744...95776...95830...95838...95893...95894...95895...95899...95906...95962...95968...95981...95985...95991...95997...96061...96085...96097...96108...96119...96121...96167...96173...96184...96229...96256...96260...96286...96290...96294...96321...96332...96355...96382...96383...96401...96402...96442...96457...96463...96483...96486...96515...96526...96533...96553...96554...96557...96578...96593...96610...96623...96645...96658...96685...96690...96691...96710...96713...96734...96747...96759...96783...96784...96830...96838...96845...96872...96874...96895...96903...96915...96923...96950...96967...96976...97006...97016...97034...97038...97039...97056...97065...97105...97106...97116...97128...97131...97157...97178...97190...97240...97249...97259...97269...97276...97299...97310...97316...97329...97336...97361...97374...97384...97389...97406...97415...97430...97460...97486...97495...97502...97551...97580...97584...97591...97632...97634...97678...97717...97718...97787...97805...97903...97912...97989...98098...98113...98131...98159...98208...98225...98227...98266...98276...98317...98335...98341...98359...98367...98415...98441...98459...98484...98509...98561...98566...98613...98641...98649...98663...98705...98723...98831...98833...98837...98844...98850...98918...98940...98968...99032...99142...99164...99238...99322...99383...99401...99419...99503...99509...99584...99617...99716...99732...99745...99768...99814...99838...99848...99895...99907...99973...100016...100051...100106...100117...100121...100173...100214...100225...100259...100286...100317...100329...100346...100374...100405...100406...100441...100484...100507...100514...100542...100605...100626...100627...100674...100717...100732...100746...100752...100808...100826...100828...100835...100922...100930...100943...100951...101002...101038...101059...101092...101143...101164...101203...101220...101234...101258...101261...101280...101284...101289...101339...101342...101357...101377...101396...101436...101455...101467...101534...101549...101561...101566...101611...101650...101658...101691...101736...101740...101748...101775...101829...101860...101893...101900...101918...101920...101942...101963...101987...102008...102017...102039...102068...102120...102128...102143...102156...102172...102246...102265...102280...102321...102356...102371...102387...102408...102438...102498...102506...102540...102570...102600...102608...102695...102700...102704...102750...102791...102797...102903...102934...102998...103070...103081...103091...103138...103168...103198...103204...103218...103221...103256...103328...103344...103346...103372...103407...103448...103511...103520...103549...103602...103608...103654...103663...103679...103729...103734...103749...103822...103854...103862...103901...103978...104056...104106...104124...104162...104212...104224...104242...104302...104319...104338...104353...104394...104401...104417...104487...104496...104524...104532...104560...104566...104605...104609...104631...104636...104643...104675...104707...104711...104719...104724...104727...104760...104772...104786...104811...104841...104845...104846...104859...104878...104880...104904...104913...104921...104934...104958...104969...104993...105023...105032...105036...105063...105065...105106...105119...105130...105148...105152...105177...105188...105202...105224...105256...105264...105265...105268...105302...105320...105331...105343...105344...105380...105386...105397...105428...105455...105498...105514...105516...105584...105612...105635...105693...105716...105828...105968...106019...106118...106168...106209...106244...106274...106296...106344...106348...106361...106373...106450...106484...106486...106535...106572...106593...106605...106650...106659...106667...106742...106743...106777...106824...106833...106865...106883...106903...106905...106916...106974...106995...107007...107037...107038...107054...107102...107107...107123...107140...107218...107247...107261...107265...107317...107331...107344...107426...107452...107461...107486...107514...107531...107593...107605...107611...107620...107656...107697...107707...107719...107720...107767...107788...107791...107800...107860...107869...107872...107895...107901...107967...107972...107975...107988...107994...108018...108045...108062...108067...108101...108132...108138...108149...108162...108183...108204...108217...108240...108256...108271...108285...108294...108314...108341...108353...108419...108420...108436...108442...108449...108494...108497...108517...108520...108532...108569...108578...108619...108631...108635...108643...108648...108676...108698...108707...108711...108722...108732...108766...108768...108790...108799...108807...108840...108850...108892...108906...108957...108963...108993...109009...109026...109037...109060...109079...109096...109113...109158...109171...109210...109216...109221...109233...109334...109335...109382...109397...109464...109472...109486...109520...109582...109585...109594...109636...109678...109683...109726...109734...109754...109809...109819...109827...109844...109921...109938...109939...110028...110041...110065...110070...110095...110104...110151...110194...110219...110285...110315...110352...110408...110435...110461...110491...110527...110547...110630...110641...110665...110693...110759...110768...110794...110817...110834...110842...110850...110881...110899...110910...110922...110930...110972...110974...111028...111033...111045...111058...111063...111067...111071...111128...111130...111148...111149...111152...111159...111225...111227...111245...111249...111274...111304...111318...111321...111322...111333...111335...111349...111364...111377...111394...111409...111445...111451...111462...111469...111471...111473...111498...111509...111553...111570...111581...111587...111589...111652...111657...111663...111664...111702...111717...111742...111770...111785...111787...111790...111793...111795...111841...111844...111862...111868...111914...111917...111930...111946...111954...111996...112008...112025...112039...112057...112074...112086...112094...112103...112130...112135...112163...112164...112172...112189...112198...112204...112224...112237...112239...112262...112265...112308...112321...112322...112363...112385...112395...112397...112401...112422...112454...112483...112493...112508...112515...112536...112547...112551...112560...112588...112625...112636...112653...112670...112685...112695...112700...112722...112729...112767...112774...112787...112800...112804...112820...112835...112854...112858...112866...112889...112898...112899...112932...112935...112948...112949...112956...112972...112973...112981...113023...113032...113045...113054...113055...113087...113101...113110...113112...113126...113156...113172...113178...113188...113212...113222...113226...113258...113270...113309...113316...113340...113384...113432...113442...113473...113528...113534...113540...113547...113577...113606...113610...113650...113665...113672...113710...113743...113757...113761...113786...113831...113909...113912...113925...113968...113985...114016...114042...114043...114092...114097...114101...114130...114151...114194...114216...114252...114279...114280...114300...114343...114348...114360...114369...114382...114454...114467...114479...114517...114555...114561...114598...114604...114623...114643...114680...114696...114749...114756...114762...114801...114830...114842...114876...114880...114889...114903...114907...114965...114969...114998...115028...115061...115087...115135...115162...115253...115269...115351...115359...115394...115458...115490...115517...115538...115610...115669...115673...115711...115741...115782...115786...115826...115832...115859...115860...115906...115915...115936...115957...115975...116005...116046...116055...116080...116101...116119...116133...116137...116138...116179...116202...116238...116261...116267...116284...116306...116340...116347...116371...116384...116408...116418...116470...116494...116501...116551...116553...116555...116579...116605...116607...116618...116624...116666...116669...116685...116689...116697...116702...116714...116738...116771...116776...116788...116829...116836...116854...116867...116872...116935...116968...116980...117002...117008...117009...117029...117046...117095...117099...117130...117132...117143...117153...117172...117178...117198...117240...117250...117261...117273...117274...117299...117302...117317...117333...117335...117350...117356...117377...117395...117401...117405...117409...117425...117434...117450...117506...117510...117518...117519...117523...117532...117540...117543...117549...117593...117605...117613...117614...117621...117627...117641...117656...117670...117678...117689...117711...117727...117752...117753...117756...117781...117782...117792...117824...117838...117847...117863...117876...117882...117923...117940...117949...117956...117974...117985...117995...118009...118022...118035...118066...118068...118069...118074...118089...118094...118099...118147...118149...118153...118164...118174...118181...118186...118209...118235...118238...118242...118248...118291...118297...118298...118322...118337...118345...118356...118363...118379...118405...118407...118429...118441...118443...118481...118499...118502...118505...118518...118524...118585...118587...118622...118633...118641...118650...118675...118723...118754...118757...118758...118810...118823...118845...118864...118882...118887...118902...118917...118923...118944...118965...118974...118985...118988...118991...119002...119039...119049...119056...119072...119096...119126...119147...119149...119156...119173...119177...119216...119234...119249...119276...119296...119300...119308...119325...119334...119360...119362...119378...119414...119419...119437...119446...119476...119515...119519...119544...119571...119605...119640...119651...119657...119675...119678...119728...119757...119776...119799...119825...119842...119869...119905...119906...119957...119989...119993...120011...120022...120026...120053...120108...120115...120154...120187...120190...120196...120238...120268...120304...120350...120353...120365...120380...120412...120445...120459...120472...120511...120543...120545...120571...120580...120662...120665...120670...120684...120734...120737...120782...120789...120798...120825...120861...120903...120911...120990...120996...121061...121115...121120...121128...121192...121202...121232...121235...121315...121316...121338...121365...121377...121397...121407...121412...121458...121461...121469...121491...121527...121559...121577...121596...121646...121663...121678...121713...121747...121772...121796...121854...121908...121925...121927...121982...122035...122042...122050...122054...122091...122134...122144...122164...122173...122178...122212...122213...122251...122273...122286...122291...122316...122340...122395...122397...122428...122472...122505...122510...122568...122623...122639...122644...122709...122718...122769...122827...122855...122915...122978...123010...123041...123047...123048...123080...123107...123119...123131...123135...123144...123191...123222...123229...123267...123273...123284...123305...123335...123352...123360...123384...123421...123430...123442...123455...123463...123471...123477...123492...123526...123528...123529...123564...123590...123608...123609...123619...123620...123653...123673...123674...123697...123699...123711...123721...123755...123768...123776...123786...123788...123800...123825...123832...123840...123848...123854...123859...123887...123914...123920...123922...123927...123959...123978...123982...124010...124035...124038...124040...124053...124075...124110...124119...124141...124159...124177...124180...124223...124227...124246...124255...124264...124302...124307...124339...124403...124406...124407...124468...124483...124487...124489...124492...124531...124576...124588...124593...124617...124639...124669...124671...124692...124733...124744...124747...124748...124760...124798...124859...124862...124885...124896...124946...124963...124994...125028...125053...125061...125091...125141...125156...125163...125165...125222...125265...125299...125315...125357...125359...125384...125426...125466...125468...125475...125517...125565...125570...125650...125680...125682...125697...125769...125778...125790...125869...125873...125908...125966...125978...125980...126012...126048...126058...126087...126107...126148...126180...126249...126257...126300...126329...126335...126339...126382...126396...126399...126422...126423...126463...126478...126526...126530...126571...126578...126590...126623...126644...126665...126717...126730...126735...126736...126788...126803...126809...126825...126861...126888...126890...126919...126933...126946...126965...126987...127015...127042...127072...127099...127103...127107...127163...127173...127181...127219...127242...127255...127354...127357...127378...127387...127438...127476...127524...127533...127577...127610...127625...127666...127669...127689...127719...127782...127794...127798...127827...127854...127866...127917...127925...127936...127938...127948...127970...127980...128001...128018...128020...128023...128026...128039...128047...128080...128119...128122...128125...128145...128154...128175...128203...128205...128215...128266...128271...128286...128287...128323...128349...128394...128405...128434...128436...128459...128470...128494...128495...128543...128570...128622...128648...128652...128662...128664...128674...128707...128747...128758...128779...128780...128786...128831...128860...128867...128872...128904...128914...128950...128952...128971...128975...129005...129026...129053...129060...129066...129069...129081...129121...129133...129148...129167...129169...129186...129202...129215...129216...129256...129268...129275...129295...129305...129314...129317...129352...129356...129393...129394...129436...129437...129463...129496...129518...129532...129543...129545...129554...129593...129599...129619...129649...129657...129718...129729...129732...129736...129757...129771...129814...129832...129836...129851...129862...129975...129991...130058...130080...130148...130150...130225...130271...130312...130402...130407...130489...130490...130535...130611...130666...130750...130782...130820...130876...130888...130928...130948...130984...131010...131032...131046...131065...131067...131081...131091...131112...131142...131186...131208...131219...131249...131267...131339...131357...131367...131370...131377...131402...131427...131471...131488...131491...131524...131584...131599...131653...131661...131679...131721...131723...131727...131753...131761...131797...131810...131837...131841...131897...131909...131916...131934...131972...131982...132016...132093...132108...132114...132176...132178...132233...132250...132267...132288...132311...132359...132389...132401...132447...132469...132488...132496...132501...132543...132570...132579...132619...132620...132647...132662...132677...132679...132723...132730...132761...132779...132790...132793...132802...132805...132809...132813...132848...132874...132882...132888...132892...132897...132915...132936...132947...132961...132963...132965...132987...132990...132991...133061...133079...133086...133090...133091...133099...133156...133165...133167...133177...133195...133218...133233...133243...133244...133259...133267...133271...133303...133330...133352...133395...133401...133406...133432...133436...133474...133509...133518...133526...133585...133587...133637...133662...133678...133713...133740...133832...133837...133851...133903...133918...133984...133987...133991...134021...134068...134089...134093...134163...134171...134242...134302...134365...134433...134435...134440...134518...134525...134651...134653...134728...134735...134806...134831...134871...134906...134932...134945...134991...135009...135058...135070...135124...135135...135154...135220...135278...135354...135368...135427...135471...135490...135560...135595...135635...135697...135719...135722...135801...135809...135816...135818...135879...135910...135922...135924...135970...136023...136062...136102...136137...136157...136213...136230...136235...136270...136274...136297...136325...136330...136350...136368...136387...136412...136457...136495...136507...136542...136579...136598...136620...136638...136688...136706...136711...136725...136770...136790...136794...136806...136845...136849...136862...136880...136911...136951...136973...136975...136977...137025...137034...137047...137048...137081...137123...137134...137136...137147...137150...137168...137176...137185...137221...137222...137225...137227...137231...137235...137298...137299...137300...137302...137303...137342...137361...137364...137373...137385...137416...137417...137419...137422...137459...137471...137474...137478...137503...137520...137551...137552...137560...137567...137585...137623...137635...137636...137657...137670...137692...137713...137742...137756...137759...137822...137862...137890...137899...137964...137995...138008...138010...138084...138105...138169...138250...138272...138338...138477...138481...138546...138628...138716...138760...138810...138819...138834...138896...138898...138959...138965...138992...139011...139015...139019...139032...139039...139108...139118...139125...139157...139170...139195...139198...139201...139230...139289...139290...139291...139325...139333...139364...139373...139408...139418...139443...139471...139472...139488...139524...139557...139558...139608...139652...139661...139662...139681...139727...139737...139772...139779...139800...139840...139841...139882...139934...139951...139970...140029...140032...140038...140112...140119...140204...140259...140272...140285...140343...140349...140364...140403...140404...140449...140465...140471...140474...140538...140556...140569...140649...140654...140662...140670...140729...140741...140778...140793...140818...140832...140845...140868...140882...140937...140954...140967...140988...141015...141035...141085...141112...141145...141175...141183...141189...141203...141218...141260...141265...141306...141337...141346...141357...141372...141377...141412...141417...141452...141458...141474...141477...141497...141518...141524...141546...141570...141608...141612...141613...141633...141652...141667...141688...141725...141737...141743...141751...141784...141785...141811...141817...141845...141861...141863...141902...141914...141915...141934...141964...141971...141975...141994...142052...142058...142076...142090...142094...142146...142149...142187...142196...142206...142226...142236...142287...142314...142328...142341...142376...142421...142423...142431...142454...142455...142503...142509...142520...142543...142567...142579...142580...142616...142671...142677...142678...142684...142688...142745...142755...142764...142786...142794...142799...142822...142833...142845...142866...142868...142881...142892...142920...142958...142986...142988...142996...143045...143050...143083...143120...143123...143129...143244...143252...143272...143273...143345...143349...143365...143386...143428...143476...143503...143564...143577...143579...143618...143637...143661...143683...143687...143743...143759...143786...143794...143815...143868...143889...143903...143909...143964...143984...143990...144050...144079...144081...144163...144174...144280...144321...144356...144407...144423...144448...144481...144525...144526...144545...144570...144613...144615...144620...144670...144689...144715...144734...144741...144749...144784...144810...144853...144857...144862...144874...144891...144910...144951...144964...144965...144966...144991...145030...145076...145077...145100...145111...145152...145154...145187...145212...145225...145231...145251...145253...145262...145299...145338...145339...145345...145348...145382...145388...145402...145438...145443...145456...145469...145475...145486...145521...145548...145553...145559...145566...145586...145601...145624...145645...145675...145684...145693...145727...145740...145750...145756...145757...145762...145825...145833...145861...145864...145872...145923...145930...145931...145970...146008...146028...146054...146078...146095...146097...146151...146165...146188...146192...146261...146264...146325...146330...146395...146404...146428...146478...146512...146572...146609...146644...146694...146696...146725...146745...146777...146784...146793...146826...146852...146858...146863...146898...146901...146927...146933...146945...146990...146991...146994...147004...147060...147062...147082...147090...147094...147109...147154...147159...147184...147187...147188...147246...147278...147283...147331...147343...147345...147361...147402...147413...147423...147457...147477...147482...147500...147507...147564...147578...147587...147588...147615...147616...147656...147669...147671...147691...147695...147735...147743...147745...147748...147766...147824...147835...147839...147883...147900...147910...147912...147919...147966...147972...147978...147996...148002...148018...148025...148040...148069...148070...148084...148098...148130...148138...148196...148197...148216...148274...148295...148309...148312...148366...148371...148389...148393...148396...148397...148450...148467...148474...148476...148514...148531...148534...148536...148549...148561...148630...148637...148666...148691...148699...148712...148735...148745...148793...148797...148819...148835...148862...148869...148871...148944...148953...148979...148989...149007...149018...149047...149092...149098...149106...149116...149182...149185...149232...149293...149309...149318...149326...149370...149409...149437...149439...149447...149483...149515...149536...149571...149575...149614...149624...149640...149651...149678...149694...149726...149742...149748...149806...149811...149831...149848...149853...149876...149878...149901...149942...149944...149953...149956...149970...149979...150014...150016...150024...150025...150049...150052...150085...150108...150129...150135...150149...150175...150205...150215...150229...150232...150243...150250...150260...150284...150305...150318...150360...150361...150388...150402...150416...150438...150458...150495...150511...150514...150527...150542...150596...150615...150617...150622...150627...150648...150714...150742...150759...150781...150822...150855...150877...150884...150896...150942...150963...150980...150992...151018...151038...151057...151093...151099...151101...151107...151156...151158...151168...151186...151191...151228...151232...151242...151260...151300...151304...151307...151309...151329...151357...151406...151416...151431...151472...151498...151511...151545...151583...151609...151622...151628...151634...151689...151696...151710...151721...151766...151775...151796...151830...151853...151873...151939...151943...151950...151983...152020...152022...152038...152109...152134...152159...152208...152224...152244...152248...152263...152280...152324...152326...152347...152348...152354...152420...152439...152452...152496...152506...152526...152553...152555...152574...152601...152663...152689...152698...152745...152746...152776...152821...152829...152837...152843...152921...152948...152967...152983...153034...153055...153078...153101...153153...153156...153160...153172...153216...153231...153235...153284...153322...153326...153339...153347...153396...153412...153413...153430...153449...153479...153483...153486...153509...153526...153541...153552...153585...153601...153609...153648...153653...153681...153696...153702...153738...153755...153788...153815...153827...153839...153886...153916...153936...153949...153957...154009...154010...154081...154087...154195...154264...154269...154284...154396...154419...154465...154511...154555...154588...154593...154658...154674...154686...154761...154774...154775...154867...154889...154890...154930...154996...155021...155027...155035...155121...155131...155271...155286...155306...155367...155377...155417...155446...155447...155503...155512...155518...155528...155556...155598...155610...155653...155671...155672...155687...155715...155771...155780...155794...155819...155864...155883...155908...155967...155972...155978...155991...156053...156075...156105...156143...156154...156213...156224...156246...156278...156285...156288...156310...156366...156371...156382...156393...156398...156402...156442...156446...156451...156456...156457...156465...156473...156509...156524...156530...156539...156548...156550...156584...156606...156633...156663...156666...156668...156725...156741...156757...156761...156777...156787...156801...156806...156817...156851...156868...156870...156884...156902...156917...156938...156987...156997...157010...157024...157082...157123...157166...157176...157202...157203...157264...157280...157283...157295...157323...157377...157382...157396...157458...157461...157491...157506...157507...157568...157573...157591...157598...157600...157631...157658...157682...157683...157718...157725...157740...157781...157799...157813...157832...157839...157891...157904...157977...158004...158085...158110...158169...158203...158232...158302...158364...158385...158413...158431...158452...158459...158517...158529...158564...158565...158599...158653...158670...158680...158701...158726...158751...158757...158784...158815...158851...158863...158876...158894...158923...158934...158988...159001...159027...159103...159112...159143...159172...159223...159257...159263...159270...159316...159358...159375...159377...159441...159448...159454...159459...159523...159532...159578...159602...159610...159624...159669...159674...159692...159693...159710...159760...159764...159780...159813...159825...159849...159852...159854...159868...159920...159921...159923...159943...159982...159995...160018...160036...160067...160128...160136...160151...160167...160176...160186...160235...160244...160255...160280...160308...160327...160336...160340...160351...160368...160370...160379...160399...160419...160424...160438...160447...160456...160459...160463...160499...160515...160536...160541...160554...160555...160559...160597...160612...160618...160624...160656...160657...160668...160683...160693...160733...160734...160746...160787...160791...160797...160813...160827...160830...160859...160860...160872...160886...160899...160912...160922...160938...160939...160990...161002...161012...161013...161035...161065...161115...161125...161130...161140...161150...161207...161210...161220...161241...161271...161313...161326...161342...161358...161361...161423...161433...161435...161457...161474...161513...161558...161560...161566...161591...161599...161654...161667...161688...161695...161697...161727...161731...161758...161769...161776...161781...161817...161822...161858...161884...161894...161908...161937...161966...161972...161991...162012...162018...162058...162071...162092...162096...162105...162156...162165...162171...162190...162191...162241...162253...162307...162316...162325...162351...162412...162433...162445...162449...162454...162484...162519...162582...162584...162594...162618...162655...162665...162709...162723...162730...162754...162787...162843...162854...162864...162902...162936...162942...162945...162988...163013...163027...163052...163057...163098...163119...163146...163202...163207...163224...163260...163295...163311...163328...163373...163405...163409...163418...163490...163497...163505...163536...163582...163602...163609...163610...163703...163749...163814...163816...163840...163852...163949...163961...163964...164046...164050...164058...164061...164128...164133...164137...164165...164225...164237...164309...164314...164360...164425...164441...164442...164501...164549...164583...164600...164677...164708...164772...164773...164800...164801...164829...164831...164847...164855...164872...164932...164950...164957...164963...164979...165007...165033...165084...165104...165126...165130...165136...165211...165215...165278...165295...165309...165345...165365...165369...165383...165406...165424...165449...165454...165486...165539...165549...165589...165630...165637...165666...165678...165738...165771...165774...165865...165877...165878...165926...165952...165959...165998...166018...166036...166043...166068...166086...166091...166125...166147...166166...166191...166228...166248...166250...166288...166295...166296...166320...166339...166358...166395...166398...166400...166417...166440...166480...166491...166511...166530...166538...166580...166597...166607...166615...166633...166665...166709...166718...166728...166747...166766...166772...166773...166788...166797...166810...166831...166879...166882...166884...166885...166887...166897...166901...166904...166964...166966...166971...166972...166976...166992...167026...167077...167101...167103...167106...167120...167135...167139...167165...167176...167177...167186...167220...167230...167249...167262...167264...167272...167299...167322...167334...167335...167382...167401...167412...167420...167480...167481...167487...167497...167515...167563...167567...167580...167593...167599...167603...167677...167688...167702...167705...167732...167739...167776...167778...167833...167836...167844...167853...167881...167903...167904...167914...167979...167983...167987...168001...168004...168038...168048...168053...168103...168132...168133...168167...168193...168242...168245...168269...168317...168323...168339...168360...168442...168453...168488...168551...168598...168671...168686...168749...168788...168831...168863...168908...168923...168993...168994...169016...169085...169092...169148...169229...169246...169265...169269...169303...169327...169333...169364...169385...169386...169399...169407...169416...169466...169481...169494...169532...169556...169573...169576...169604...169623...169630...169657...169687...169693...169695...169719...169722...169747...169772...169774...169778...169804...169814...169838...169865...169872...169880...169881...169888...169972...169973...169976...169991...170029...170055...170061...170081...170128...170161...170190...170246...170253...170288...170330...170331...170393...170418...170438...170471...170477...170505...170522...170545...170591...170603...170617...170619...170705...170736...170754...170762...170767...170768...170814...170827...170843...170851...170867...170883...170903...170917...170954...170957...171009...171066...171080...171082...171083...171105...171182...171208...171212...171223...171248...171292...171313...171345...171357...171359...171378...171402...171411...171427...171454...171460...171465...171477...171505...171547...171560...171561...171604...171609...171610...171629...171673...171725...171738...171740...171758...171781...171795...171826...171886...171956...171974...172020...172048...172085...172106...172128...172176...172187...172241...172263...172271...172277...172336...172356...172394...172407...172410...172468...172495...172496...172537...172560...172579...172587...172620...172646...172653...172726...172750...172760...172812...172845...172859...172885...172916...172920...172972...172986...173012...173048...173083...173107...173119...173152...173158...173216...173240...173270...173277...173299...173303...173342...173355...173357...173378...173419...173456...173467...173479...173487...173532...173555...173591...173598...173628...173633...173671...173676...173705...173707...173779...173780...173810...173820...173875...173894...173901...173957...173961...173972...174022...174030...174033...174092...174103...174126...174176...174190...174240...174260...174269...174283...174323...174384...174440...174478...174546...174549...174579...174614...174638...174639...174692...174736...174782...174821...175171...175222...175245...175320...175347...175376...175382...175386...175466...175471...175484...175486...175513...175548...175572...175597...175605...175625...175653...175716...175727...175737...175764...175798...175803...175837...175849...175858...175881...175913...175943...175963...176005...176011...176024...176063...176085...176109...176160...176188...176193...176249...176277...176303...176323...176338...176390...176412...176426...176445...176521...176537...176553...176554...176579...176624...176635...176681...176682...176724...176733...176748...176752...176760...176816...176829...176847...176848...176867...176915...176917...176919...176947...176964...176974...176989...176997...177003...177023...177039...177045...177066...177090...177095...177100...177150...177200...177210...177213...177217...177222...177280...177281...177293...177318...177336...177379...177396...177404...177421...177433...177463...177465...177486...177504...177506...177539...177560...177570...177583...177602...177632...177636...177652...177676...177692...177709...177712...177714...177735...177770...177797...177806...177827...177841...177852...177872...177908...177931...177936...177945...177980...178012...178017...178094...178101...178115...178134...178138...178171...178172...178191...178205...178253...178268...178284...178297...178304...178322...178340...178361...178380...178400...178403...178417...178440...178445...178481...178492...178503...178510...178529...178530...178552...178561...178565...178574...178582...178622...178639...178661...178682...178709...178712...178716...178744...178762...178774...178813...178821...178825...178866...178871...178883...178932...178934...178945...178953...178974...178992...179041...179088...179107...179110...179121...179132...179186...179224...179225...179227...179255...179264...179275...179294...179318...179338...179351...179358...179380...179384...179405...179438...179440...179446...179462...179552...179572...179589...179614...179666...179668...179718...179760...179774...179793...179804...179838...179915...179923...179940...180025...180041...180102...180122...180195...180216...180303...180324...180388...180398...180455...180457...180466...180486...180558...180586...180607...180623...180664...180669...180727...180774...180792...180871...180885...180964...181005...181040...181169...181301...181362...181374...181375...181421...181442...181445...181453...181512...181513...181514...181518...181524...181536...181573...181597...181605...181607...181621...181640...181643...181647...181702...181715...181726...181728...181729...181731...181779...181798...181819...181842...181869...181887...181904...181907...181955...181961...182004...182030...182045...182048...182088...182113...182129...182136...182174...182212...182223...182231...182259...182293...182310...182320...182350...182365...182377...182405...182418...182423...182429...182473...182482...182486...182489...182544...182561...182562...182568...182634...182646...182654...182657...182661...182709...182732...182752...182793...182798...182799...182822...182856...182876...182891...182892...182939...182946...182954...182978...182985...182994...183035...183040...183060...183076...183097...183107...183129...183159...183209...183226...183230...183231...183236...183242...183288...183298...183304...183344...183351...183369...183374...183451...183456...183458...183463...183526...183541...183594...183619...183640...183642...183674...183702...183715...183733...183738...183789...183790...183818...183827...183848...183850...183869...183879...183900...183934...183939...183972...183973...183979...183997...184016...184037...184051...184077...184088...184098...184115...184119...184154...184170...184175...184194...184237...184250...184268...184324...184332...184342...184365...184385...184390...184401...184410...184435...184465...184474...184503...184518...184540...184545...184555...184590...184608...184614...184618...184673...184700...184750...184756...184798...184830...184873...184874...184903...184931...184964...184984...184995...185026...185038...185045...185052...185090...185120...185126...185128...185129...185163...185164...185167...185225...185253...185255...185295...185298...185348...185362...185371...185382...185387...185423...185444...185459...185460...185470...185499...185502...185574...185630...185644...185656...185669...185730...185748...185779...185831...185852...185867...185880...185921...185925...185987...185991...185997...186004...186006...186061...186084...186125...186128...186130...186159...186165...186169...186189...186231...186239...186250...186252...186254...186264...186265...186308...186313...186321...186329...186334...186335...186352...186397...186402...186432...186437...186441...186448...186452...186477...186521...186524...186544...186586...186590...186635...186650...186653...186659...186671...186740...186742...186774...186785...186822...186838...186839...186844...186849...186866...186902...186918...186929...186947...186955...186973...186999...187014...187041...187058...187084...187095...187104...187144...187165...187178...187187...187235...187250...187276...187278...187279...187350...187358...187412...187443...187488...187541...187573...187648...187670...187701...187714...187792...187796...187839...187893...187909...187937...187991...188016...188061...188107...188110...188139...188161...188235...188242...188274...188333...188343...188380...188430...188443...188455...188526...188547...188625...188690...188742...188749...188752...188849...188868...188881...188938...188987...188995...189072...189109...189120...189236...189244...189257...189269...189285...189318...189346...189366...189369...189428...189436...189451...189453...189466...189505...189536...189543...189560...189570...189598...189609...189613...189640...189667...189674...189685...189756...189776...189796...189835...189837...189854...189919...189921...189929...189936...189981...189994...190002...190003...190007...190008...190031...190054...190080...190098...190112...190125...190128...190143...190173...190182...190187...190211...190220...190226...190253...190268...190275...190311...190333...190334...190387...190394...190400...190433...190445...190469...190479...190511...190530...190571...190585...190592...190607...190611...190647...190661...190685...190688...190694...190729...190735...190751...190813...190817...190826...190845...190848...190893...190900...190903...190919...190931...190974...190979...190982...191002...191038...191042...191053...191069...191074...191108...191119...191124...191126...191185...191192...191196...191206...191254...191278...191294...191326...191357...191372...191416...191417...191441...191482...191483...191501...191522...191569...191571...191612...191653...191679...191687...191723...191731...191803...191806...191879...191916...191968...191969...192008...192072...192088...192122...192170...192192...192208...192211...192218...192277...192290...192306...192321...192355...192357...192389...192394...192420...192426...192478...192500...192539...192544...192595...192606...192611...192638...192670...192692...192695...192704...192767...192796...192809...192824...192844...192855...192885...192903...192932...192970...192982...192983...193009...193039...193047...193049...193073...193082...193084...193117...193145...193150...193160...193165...193237...193264...193300...193304...193342...193352...193362...193379...193428...193437...193448...193451...193514...193519...193528...193550...193559...193579...193594...193595...193604...193641...193643...193651...193689...193716...193726...193727...193743...193774...193787...193840...193854...193859...193862...193871...193923...193935...193938...193953...193958...193959...193997...194019...194028...194032...194051...194089...194092...194109...194112...194154...194162...194163...194180...194196...194242...194254...194271...194276...194309...194327...194336...194340...194354...194365...194398...194418...194450...194453...194467...194480...194482...194552...194559...194603...194613...194673...194679...194682...194693...194760...194772...194778...194865...194870...194882...194921...194981...194983...194993...195011...195047...195076...195079...195089...195132...195140...195211...195239...195254...195274...195289...195324...195349...195410...195421...195478...195504...195546...195566...195713...195759...195787...195797...195836...195861...195890...195921...195953...196003...196020...196072...196091...196098...196175...196181...196192...196226...196246...196250...196251...196310...196324...196341...196390...196428...196442...196484...196528...196534...196551...196569...196603...196652...196668...196688...196725...196743...196793...196806...196818...196819...196863...196916...196935...196960...196975...197028...197045...197049...197062...197127...197135...197152...197156...197219...197226...197269...197271...197281...197325...197361...197364...197384...197404...197448...197459...197499...197510...197541...197556...197558...197559...197588...197622...197641...197670...197684...197691...197706...197722...197753...197794...197796...197801...197803...197833...197836...197851...197869...197876...197890...197911...197922...197930...197934...197942...197954...197985...197996...198019...198033...198052...198059...198077...198086...198107...198126...198130...198149...198156...198172...198198...198217...198227...198233...198242...198270...198274...198276...198297...198301...198313...198316...198335...198359...198360...198362...198385...198387...198402...198440...198462...198466...198483...198499...198506...198537...198554...198560...198566...198591...198636...198648...198684...198693...198711...198720...198734...198813...198827...198834...198866...198870...198875...198885...198921...198942...198956...198970...198996...199035...199073...199095...199099...199112...199152...199163...199174...199187...199247...199261...199307...199320...199371...199374...199375...199386...199387...199455...199482...199491...199549...199561...199570...199630...199645...199659...199683...199687...199727...199762...199766...199773...199795...199810...199828...199851...199871...199889...199893...199939...199941...199959...199968...199972...199987...200050...200065...200070...200072...200074...200100...200124...200157...200186...200216...200224...200238...200273...200294...200315...200397...200404...200475...200535...200585...200616...200690...200773...200905...201008...201102...201183...201269...201284...201315...201347...201394...201438...201453...201497...201536...201582...201617...201641...201702...201721...201724...201767...201791...201805...201814...201831...201844...201863...201878...201910...201927...201944...201957...201967...201975...201989...202007...202019...202034...202036...202049...202077...202099...202133...202140...202148...202149...202222...202223...202236...202244...202258...202267...202272...202305...202356...202363...202374...202388...202426...202434...202456...202459...202488...202496...202515...202526...202569...202578...202589...202630...202635...202643...202662...202685...202708...202720...202754...202768...202781...202789...202798...202800...202825...202834...202849...202857...202872...202882...202895...202923...202927...202938...202953...202958...202970...202983...203003...203011...203012...203020...203031...203042...203049...203064...203067...203085...203105...203116...203122...203124...203131...203147...203161...203167...203196...203207...203208...203209...203211...203227...203247...203261...203265...203276...203281...203307...203315...203320...203336...203370...203380...203389...203401...203403...203420...203437...203456...203471...203489...203504...203509...203524...203555...203567...203569...203575...203583...203598...203601...203645...203649...203673...203675...203722...203735...203778...203783...203802...203854...203856...203862...203887...203927...203937...203974...204017...204029...204088...204099...204101...204197...204283...204288...204289...204406...204474...204488...204511...204578...204583...204644...204659...204678...204730...204758...204839...204846...204927...204934...204943...205007...205069...205071...205094...205139...205143...205213...205221...205226...205282...205295...205299...205359...205377...205400...205420...205455...205494...205508...205544...205609...205744...205776...205825...205836...205858...205863...205902...205914...205929...205940...205969...205998...206015...206017...206022...206075...206089...206106...206164...206172...206174...206199...206230...206255...206274...206302...206318...206325...206330...206363...206372...206408...206425...206452...206454...206490...206502...206503...206517...206526...206538...206567...206586...206597...206610...206613...206639...206644...206653...206655...206660...206683...206721...206734...206738...206753...206763...206778...206805...206813...206814...206832...206845...206873...206876...206887...206891...206930...206938...206947...206949...206967...206971...206984...206996...207007...207009...207021...207024...207055...207078...207080...207096...207102...207169...207170...207173...207197...207210...207230...207253...207286...207291...207297...207298...207307...207327...207360...207384...207398...207404...207405...207413...207419...207420...207466...207508...207511...207517...207531...207535...207547...207572...207604...207641...207643...207652...207678...207688...207720...207739...207754...207795...207804...207828...207840...207854...207883...207894...207916...207936...207943...207960...207979...207991...207995...208023...208029...208065...208125...208126...208166...208172...208176...208178...208201...208248...208260...208290...208294...208298...208299...208336...208355...208362...208379...208403...208433...208434...208469...208476...208482...208500...208507...208532...208619...208621...208622...208712...208719...208747...208751...208799...208811...208820...208828...208876...208937...208949...208951...209032...209068...209076...209100...209194...209198...209202...209283...209326...209340...209393...209405...209426...209484...209514...209605...209619...209663...209665...209681...209697...209729...209737...209746...209748...209763...209784...209789...209803...209818...209824...209835...209941...209947...209965...209993...210012...210037...210046...210052...210078...210113...210116...210122...210132...210162...210199...210246...210251...210260...210276...210327...210340...210344...210393...210404...210425...210447...210490...210491...210512...210597...210608...210611...210613...210696...210716...210721...210769...210798...210862...210886...210894...210915...210944...210996...211015...211077...211080...211085...211088...211142...211157...211164...211177...211181...211205...211217...211246...211257...211275...211278...211310...211313...211315...211366...211369...211377...211387...211389...211412...211422...211427...211463...211474...211475...211485...211493...211544...211546...211553...211554...211570...211597...211629...211644...211653...211656...211702...211733...211737...211767...211780...211836...211838...211862...211882...211889...211915...211936...211982...211983...212009...212028...212049...212100...212121...212122...212126...212130...212187...212222...212232...212239...212252...212288...212330...212360...212365...212391...212425...212432...212464...212484...212489...212527...212559...212567...212604...212610...212643...212651...212654...212660...212694...212744...212765...212768...212769...212786...212824...212837...212840...212851...212914...212951...212960...213022...213047...213069...213115...213128...213178...213186...213195...213204...213226...213255...213266...213274...213283...213303...213338...213344...213357...213374...213407...213429...213430...213447...213465...213476...213523...213532...213542...213549...213561...213591...213614...213619...213635...213654...213666...213706...213720...213736...213752...213794...213828...213833...213834...213856...213880...213904...213917...213952...213979...213996...214010...214019...214023...214035...214073...214082...214092...214098...214118...214148...214156...214177...214200...214225...214238...214258...214260...214293...214335...214339...214340...214395...214400...214423...214427...214449...214469...214481...214489...214501...214552...214588...214629...214638...214667...214680...214685...214686...214727...214732...214770...214794...214843...214850...214897...214911...214915...214920...214964...214970...214974...214987...215045...215084...215098...215149...215163...215215...215234...215242...215311...215313...215321...215328...215385...215484...215502...215510...215590...215605...215629...215644...215695...215724...215729...215733...215747...215819...215831...215855...215874...215888...215895...215948...216016...216021...216024...216038...216084...216103...216122...216127...216174...216202...216252...216292...216309...216329...216385...216395...216404...216455...216529...216549...216625...216676...216699...216731...216795...216834...216896...216940...216948...217044...217058...217091...217122...217144...217166...217264...217292...217310...217317...217333...217353...217373...217397...217398...217426...217477...217486...217490...217515...217524...217541...217564...217587...217604...217651...217656...217665...217667...217668...217678...217680...217733...217739...217743...217768...217784...217799...217803...217812...217815...217821...217864...217885...217890...217915...217922...217951...217968...217974...218022...218049...218078...218133...218140...218185...218200...218278...218299...218319...218390...218411...218412...218417...218486...218488...218508...218545...218590...218618...218619...218679...218691...218719...218727...218760...218827...218858...218864...218885...218932...218952...218961...219030...219031...219061...219110...219125...219144...219254...219256...219382...219398...219411...219456...219495...219542...219560...219591...219630...219634...219674...219726...219732...219733...219734...219751...219769...219782...219796...219813...219819...219833...219848...219865...219879...219903...219910...219913...219919...219921...219926...219980...219986...219995...219999...220015...220033...220034...220039...220045...220081...220086...220099...220105...220108...220130...220147...220184...220190...220193...220213...220215...220249...220264...220331...220337...220347...220351...220366...220368...220409...220451...220477...220524...220529...220577...220617...220654...220656...220680...220714...220743...220762...220848...220895...220950...221044...221063...221145...221154...221203...221235...221317...221332...221373...221410...221423...221510...221524...221543...221572...221600...221625...221638...221666...221687...221716...221735...221741...221749...221758...221800...221825...221858...221869...221881...221900...221902...221915...221981...221990...222004...222034...222038...222073...222074...222075...222080...222105...222135...222151...222158...222162...222189...222217...222220...222232...222246...222267...222275...222314...222322...222330...222333...222364...222405...222423...222435...222465...222498...222506...222520...222524...222539...222579...222582...222602...222616...222648...222659...222664...222746...222767...222771...222807...222821...222832...222863...222894...222919...222926...222930...223002...223030...223032...223056...223100...223110...223169...223189...223201...223215...223240...223254...223285...223304...223330...223334...223349...223363...223389...223399...223426...223432...223444...223457...223468...223494...223503...223512...223530...223550...223577...223585...223591...223599...223619...223636...223645...223649...223653...223655...223666...223686...223695...223723...223727...223734...223741...223746...223771...223785...223805...223814...223823...223838...223847...223884...223892...223907...223920...223931...223951...223969...223984...223997...224035...224048...224073...224089...224092...224102...224166...224175...224197...224199...224242...224273...224291...224314...224345...224350...224373...224454...224475...224500...224518...224521...224550...224644...224646...224673...224761...224774...224827...224884...224892...224900...224916...224976...224978...225000...225013...225087...225113...225128...225159...225204...225227...225248...225271...225326...225336...225396...225401...225448...225468...225475...225484...225531...225550...225596...225607...225623...225637...225674...225700...225708...225713...225720...225742...225765...225782...225784...225789...225822...225835...225853...225867...225870...225889...225928...225931...225943...225947...225948...225992...225998...226014...226016...226024...226036...226047...226073...226076...226103...226117...226164...226193...226212...226234...226257...226274...226289...226298...226303...226347...226363...226370...226409...226432...226436...226437...226476...226519...226520...226524...226529...226584...226594...226627...226642...226648...226708...226731...226732...226737...226778...226783...226791...226816...226848...226861...226866...226879...226883...226927...226973...227027...227064...227094...227099...227116...227155...227200...227211...227226...227295...227384...227479...227488...227565...227589...227615...227682...227714...227724...227831...227845...227859...227911...227918...227937...227951...227953...227985...228010...228022...228025...228035...228050...228066...228081...228082...228093...228133...228138...228148...228179...228189...228199...228202...228207...228226...228256...228277...228306...228311...228320...228329...228364...228404...228416...228428...228451...228486...228490...228529...228533...228541...228556...228617...228638...228662...228665...228727...228746...228751...228802...228818...228856...228927...228938...228943...228956...229007...229031...229035...229047...229068...229105...229106...229109...229133...229160...229232...229240...229249...229302...229305...229316...229328...229375...229396...229402...229405...229406...229427...229455...229476...229505...229506...229530...229548...229555...229579...229600...229612...229613...229633...229638...229682...229694...229696...229716...229726...229748...229770...229779...229795...229810...229817...229819...229823...229869...229880...229881...229884...229890...229897...229902...229974...229979...229984...229994...229999...230033...230072...230075...230081...230088...230092...230132...230162...230185...230194...230222...230252...230263...230321...230330...230377...230381...230412...230442...230465...230466...230497...230507...230550...230625...230644...230703...230748...230779...230783...230875...230886...230903...230961...230997...231072...231214...231292...231390...231394...231408...231479...231507...231510...231552...231582...231596...231625...231645...231664...231672...231703...231705...231726...231733...231747...231776...231792...231807...231829...231947...231950...231952...231985...232026...232060...232064...232102...232111...232138...232152...232201...232213...232223...232225...232246...232264...232269...232293...232343...232347...232367...232388...232410...232431...232461...232470...232513...232524...232542...232588...232593...232609...232611...232612...232649...232672...232698...232743...232745...232747...232751...232790...232822...232835...232842...232873...232907...232924...232941...232957...232975...232976...233048...233067...233083...233084...233087...233120...233142...233153...233167...233172...233199...233260...233263...233266...233280...233284...233339...233345...233369...233415...233435...233446...233453...233456...233463...233498...233517...233523...233546...233556...233564...233604...233608...233618...233626...233645...233670...233695...233728...233761...233776...233786...233803...233826...233843...233846...233891...233897...233925...233928...233936...233938...233980...234012...234019...234031...234054...234086...234120...234129...234141...234202...234208...234237...234244...234246...234275...234300...234324...234353...234371...234395...234399...234410...234415...234425...234477...234543...234545...234596...234597...234614...234664...234709...234781...234783...234799...234857...234891...234892...234915...234989...234990...234997...235006...235084...235087...235110...235131...235145...235228...235248...235257...235264...235269...235311...235370...235379...235405...235461...235483...235514...235516...235517...235574...235598...235613...235627...235634...235664...235675...235706...235737...235780...235791...235817...235818...235833...235859...235900...235918...235920...235923...235984...236006...236009...236053...236058...236106...236187...236189...236234...236265...236279...236293...236311...236316...236351...236374...236382...236423...236450...236468...236507...236520...236602...236605...236651...236654...236670...236679...236687...236688...236719...236722...236761...236771...236782...236792...236810...236853...236854...236897...236913...236922...236974...236981...237001...237054...237066...237106...237113...237116...237149...237178...237199...237239...237260...237268...237269...237277...237288...237311...237375...237385...237390...237394...237469...237482...237490...237509...237542...237566...237568...237573...237594...237655...237663...237674...237684...237719...237762...237801...237835...237860...237885...237897...237926...237932...237958...238015...238017...238029...238043...238056...238061...238089...238121...238136...238152...238164...238176...238225...238228...238234...238238...238250...238260...238277...238337...238346...238351...238364...238367...238415...238430...238432...238445...238465...238475...238494...238539...238554...238558...238568...238620...238632...238635...238651...238696...238724...238726...238733...238745...238752...238788...238812...238817...238833...238880...238886...238890...238930...238963...238998...239007...239013...239026...239033...239078...239112...239114...239128...239144...239146...239181...239219...239221...239225...239251...239278...239285...239338...239339...239342...239345...239382...239383...239406...239430...239439...239453...239456...239481...239513...239533...239544...239553...239562...239580...239610...239632...239634...239639...239640...239650...239661...239712...239716...239730...239745...239761...239799...239814...239863...239875...239898...239924...239979...240012...240013...240056...240113...240136...240208...240231...240305...240324...240383...240430...240445...240522...240529...240536...240607...240623...240625...240650...240689...240707...240736...240748...240769...240779...240812...240850...240859...240860...240899...240906...240968...240982...241003...241063...241104...241140...241183...241242...241277...241355...241363...241452...241483...241514...241554...241590...241601...241628...241666...241694...241737...241763...241772...241800...241909...241976...241977...242050...242054...242088...242139...242157...242223...242228...242246...242315...242330...242335...242347...242420...242430...242464...242484...242506...242512...242556...242566...242575...242598...242606...242612...242645...242666...242675...242678...242693...242695...242697...242710...242739...242742...242744...242763...242764...242769...242779...242827...242829...242877...242901...242909...242974...243001...243003...243005...243041...243058...243072...243084...243091...243137...243187...243198...243206...243210...243218...243219...243265...243278...243279...243281...243291...243292...243306...243325...243337...243366...243373...243398...243406...243422...243436...243446...243456...243461...243490...243492...243501...243522...243542...243551...243555...243559...243563...243564...243578...243580...243593...243614...243622...243629...243659...243673...243688...243689...243706...243708...243728...243739...243743...243749...243776...243797...243812...243815...243826...243832...243906...243907...243908...243934...243940...243953...244005...244011...244013...244048...244077...244142...244173...244200...244238...244317...244428...244522...244609...244820...244892...244961...245060...245152...245298...245324...245417...245423...245430...245503...245506...245520...245598...245623...245627...245630...245693...245708...245712...245755...245795...245801...245845...245846...245906...245910...246003...246009...246087...246125...246191...246212...246235...246317...246335...246342...246365...246416...246421...246457...246478...246509...246512...246516...246574...246576...246581...246589...246595...246614...246630...246659...246690...246702...246730...246743...246744...246784...246799...246813...246835...246841...246855...246861...246876...246889...246897...246947...246951...246952...246974...246979...246995...247038...247039...247041...247050...247079...247082...247095...247106...247120...247132...247144...247154...247172...247182...247205...247207...247219...247221...247272...247296...247297...247318...247742...248005...248247...248339...248517...248636...249173...249185...249212...249214...249865...All jobs completed. Finish execution
Final value of timer: 250400.000000 seconds (69.555556 hours)
Time when number of pending job is zero: 247205.000000 seconds (68.668056 hours)

Plot preemption vectors for different policies as heatmaps

  • If time_limit is set (the value can be copied from the output of the previous cell), it will help exclude the tail end of execution when not all of the nodes are utilized (i.e., number of pending jobs is zero); for consistency reasons, such tail end should not be compared to the rest of the experiment when all nodes are running jobs.
In [14]:
preemptions = pd.read_csv("preemptions.csv")
# Exclude index column recorded in the file
preemptions.drop(["Unnamed: 0"], inplace=True, axis=1)

# Plot data only up to this time limit (in seconds); if not set, trimming will not be enforced
time_limit = 246835.000000

# Trim to the limit
if time_limit:
    preemptions = preemptions[preemptions["Time"] <= time_limit]

node_count = len([node for node in preemptions.columns.values if "node-" in node])

grouped_preemptions = preemptions.groupby(["Policy"])

for p, group in grouped_preemptions:

    # rename columns: Node-X -> X
    column_name_map = {"node-%d" % id: id for id in range(node_count)}
    x = group.rename(columns=column_name_map)
    # print x.head()

    # drop unnecessary column
    x = x.drop(['Policy'], axis=1)
    # print x.head(20)
    
    # Convert time from seconds to hours for cleaner plotting
    x['Time'] = x['Time']/3600.0
    
    xticklabel_count = 8
    xticklabel_interval = int(len(x)/xticklabel_count)

    x = pd.melt(x, id_vars=['Time'], var_name='Node')
    
    x = x.pivot("Node", "Time", "value")

    fig, ax = plt.subplots(1,1, figsize=FSIZE)
    
    sns.heatmap(x, cmap="YlOrBr", xticklabels=xticklabel_interval, ax=ax, cbar_kws={"label": "Preemption Value"})
    # sns.heatmap(x, cmap="YlOrBr", ax=ax, cbar_kws={"label": "Preemption Value"})
    
    labels = ["%.1f" % float(item.get_text()) for item in ax.get_xticklabels()]
    ax.set_xticklabels(labels)
    ax.set_xlabel('Time, hours')
    
    ax.set_title("Runtime-based Preemption Vectors. Workload: %s. Policy: %s" % (MODE.upper(),p))
    ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=0)
    fig.tight_layout()
    fig.savefig(p + '-heatmap.png', dpi=300)

Create timeline and violin plots for wasted cycles

  • Same as above, if time_limit is set, the tail end of execution when not all of the nodes are utilized will be excluded.
In [21]:
wasted_cycles = pd.read_csv("wasted_cycles.csv")
wasted_cycles.drop(["Unnamed: 0"], inplace=True, axis=1)

# Plot data only up to this time limit (in seconds); if not set, trimming will not be enforced
time_limit = 246835.000000

# If HTC, exclude PAP policy because it is the same as LIFO
if MODE == "htc":
    wasted_cycles = wasted_cycles[wasted_cycles["Policy"] != "PAP"]

sns.set_style("white")
sns.set_palette(sns.color_palette("Paired"))

gp_vals = sorted(wasted_cycles["GracePeriod"].unique().tolist())
#norm = plt.Normalize()
#gp_cols = plt.cm.jet(norm(range(len(gp_vals))))
gp_cols = sns.color_palette(n_colors=4) 

grouped_wc = wasted_cycles.groupby(["Policy"])

violin_data = pd.DataFrame()
for p, p_group in grouped_wc:
    # Loop over different policies
    fig, ax = plt.subplots(figsize=(10,4))
    
    col_id = 0
    for g, pg_group in p_group.groupby(["GracePeriod"]):
        # Loop over different grace periods within the same policy
        for_plot=pg_group.sort_values(by=['Time'])
        # Trim to the limit
        if time_limit:
            for_plot = for_plot[for_plot["Time"] <= time_limit]
        
        # Convert time from seconds to hours for cleaner plotting
        for_plot['Time'] = for_plot['Time']/3600.0
        # Convert wasted cycles from seconds to hours for cleaner plotting
        for_plot['WastedCycles'] = for_plot['WastedCycles']/3600.0
        
        for_plot.plot(kind='line', x='Time', y='WastedCycles', \
                           c=gp_cols[col_id],label='%s-%d' % (p,g), ax = ax)
        col_id += 1
         
        #violin_data["%s-%d" %(p, g)] = pg_group["WastedCycles"].reset_index(drop=True)
        #violin_data = violin_data.append(pg_group[["WastedCycles", "Policy", "GracePeriod"]])
        
        violin_data = violin_data.append(pg_group[["WastedCycles", "Policy", "GracePeriod"]])
        
        print "Policy: %s, GracePeriod: %s, # of samples: %d, avg. time between samples in seconds: %f" % (p, g, len(for_plot), for_plot['Time'].max() * 3600.0 / len(for_plot))
         
        
    ax.set_title("Wasted Cycles. Workload: %s. Policy: %s" % (MODE.upper(),p))
    ax.set_ylabel("Wasted Cycles, hours")
    ax.set_xlabel("Time, hours")
    ax.legend(loc="upper left") 
    
    fig.savefig(p + '-timeline.png', dpi=300)

# Convert wasted cycles from seconds to hours for cleaner plotting
violin_data['WastedCycles'] = violin_data['WastedCycles']/3600.0

if MODE == "htc":
    # Displaying 3 out 4 policies
    fig, ax = plt.subplots(figsize=(FSIZE[0]*(3.0/4),FSIZE[1]))
else:
    # In cases other than htc, PAP is shown and the graph needs to have full width
    fig, ax = plt.subplots(figsize=FSIZE)
    
sns.set_style("ticks")
sns.violinplot(data=violin_data, x="Policy", y="WastedCycles", hue="GracePeriod", cut=0, ax=ax);
#sns.despine(trim=True)
#for label in ax.get_xticklabels():
#    label.set_rotation(45)  
ax.set_title("Comparison of Policies and Grace Periods. Workload: %s" % (MODE.upper()))
ax.set_ylabel("Wasted Cycles, hours")
ax.legend(title="Grace Period, s", loc="upper center", ncol=4)

# No reason to show negative values
_,ylim_ub = ax.get_ylim()
ax.set_ylim(0,ylim_ub)

fig.tight_layout()
fig.savefig( 'wasted_cycles-violins.png', dpi=300)
Policy: FIFO, GracePeriod: 60, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: FIFO, GracePeriod: 120, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: FIFO, GracePeriod: 1200, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: FIFO, GracePeriod: 1800, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: LIFO, GracePeriod: 60, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: LIFO, GracePeriod: 120, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: LIFO, GracePeriod: 1200, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: LIFO, GracePeriod: 1800, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: RANDOM, GracePeriod: 60, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: RANDOM, GracePeriod: 120, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: RANDOM, GracePeriod: 1200, # of samples: 14605, avg. time between samples in seconds: 16.900719
Policy: RANDOM, GracePeriod: 1800, # of samples: 14605, avg. time between samples in seconds: 16.900719