
    j"$                         d Z ddlZddlZddlZddlmZ ddlmZ ddlmZ ddl	Z	 G d d          Z
d Zed	k    r e             dS dS )
a  
Collect Binance order book snapshots for multiple assets.

This script:
1. Connects to Binance WebSocket API
2. Collects real-time order book depth for multiple symbols
3. Stores bid/ask volumes at each price level
4. Saves to SQLite for order flow graph processing
    N)Path)datetime)defaultdictc                   b    e Zd ZdZdedefdZd Zdede	de	fd	Z
d
 Zd Zd Zd Zdde	fdZdS )BinanceMultiAssetCollectorz>Collect real-time Binance order book data for multiple assets.symbols	output_dbc                     d |D             | _         || _        d | j         D             | _        d | j         D             | _        d | j         D             | _        d| _        d S )Nc                 6    g | ]}|                                 S  )lower.0ss     W/home/ubuntu/.hermes/workspace/projects/ORDER_FLOW_GRAPH/scripts/collect_multi_asset.py
<listcomp>z7BinanceMultiAssetCollector.__init__.<locals>.<listcomp>   s     333a		333    c                     i | ]}|i i d 	S ))bidsasksr   r   symbols     r   
<dictcomp>z7BinanceMultiAssetCollector.__init__.<locals>.<dictcomp>   s"    XXXFR$<$<XXXr   c                     i | ]}|d S r   r   r   s     r   r   z7BinanceMultiAssetCollector.__init__.<locals>.<dictcomp>       EEEfEEEr   c                     i | ]}|d S r   r   r   s     r   r   z7BinanceMultiAssetCollector.__init__.<locals>.<dictcomp>   r   r   r   )r   r	   order_bookslast_update_idssnapshot_countssnapshot_id)selfr   r	   s      r   __init__z#BinanceMultiAssetCollector.__init__   st    337333"XX4<XXXEEEEEEEEEEr   c                 |   t          d| j         d           t          j        | j                  }|                                }|                    d           |                    d           |                    d           |                                 |                                 t          d           dS )z0Create SQLite database for order book snapshots.zCreating database at ...a  
            CREATE TABLE IF NOT EXISTS order_book_snapshots (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                symbol TEXT NOT NULL,
                timestamp INTEGER NOT NULL,
                update_id INTEGER NOT NULL,
                bid_levels INTEGER NOT NULL,
                ask_levels INTEGER NOT NULL,
                total_bid_volume REAL NOT NULL,
                total_ask_volume REAL NOT NULL,
                created_at INTEGER NOT NULL
            )
        as  
            CREATE TABLE IF NOT EXISTS order_book_levels (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                snapshot_id INTEGER NOT NULL,
                side TEXT NOT NULL,
                price REAL NOT NULL,
                volume REAL NOT NULL,
                FOREIGN KEY (snapshot_id) REFERENCES order_book_snapshots(id)
            )
        z|
            CREATE INDEX IF NOT EXISTS idx_symbol_timestamp
            ON order_book_snapshots(symbol, timestamp)
        u     ✓ Database createdN)printr	   sqlite3connectcursorexecutecommitclose)r"   connr)   s      r   create_databasez*BinanceMultiAssetCollector.create_database    s    9dn999:::t~.. 	  	 	 	 	 	 		 		 		 	  	 	 	
 	

&'''''r   r   	timestampreturnc                    t          j        | j                  }|                                }| j        |         }t          |d                                                   }t          |d                                                   }t          |d                   }t          |d                   }	|                    d|	                                || j
        |         ||	||t          t          j                              f           |j        }
|d                                         D ]\  }}|                    d|
d||f            |d                                         D ]\  }}|                    d|
d||f            |                                 |                                 | j        |xx         dz  cc<   |
S )z*Save current order book state to database.r   r   z
            INSERT INTO order_book_snapshots
            (symbol, timestamp, update_id, bid_levels, ask_levels, total_bid_volume, total_ask_volume, created_at)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        z
                INSERT INTO order_book_levels (snapshot_id, side, price, volume)
                VALUES (?, ?, ?, ?)
            bidask   )r'   r(   r	   r)   r   sumvalueslenr*   upperr   inttime	lastrowiditemsr+   r,   r    )r"   r   r/   r-   r)   
order_booktotal_bid_volumetotal_ask_volume
bid_levels
ask_levelsr!   pricevolumes                r   save_snapshotz(BinanceMultiAssetCollector.save_snapshotM   s   t~..%f-
 z&188::;;z&188::;;F+,,
F+,,
 	 
 LLNN (		
		 	 	 & (/5577 	6 	6ME6NN  ueV46 6 6 6 (/5577 	6 	6ME6NN  ueV46 6 6 6
 	

V$$$)$$$r   c                 (   	 t          j        |          }d|v rEd|v rB|d         }|                    d          d         }|| j        vrdS |d         }d|v r	|d         dk    r|d         | j        |<   | j        |         }|d	         D ]H\  }}	t          |          }
t          |	          }|dk    r|
|d
         v r	|d
         |
= =||d
         |
<   I|d         D ]H\  }}	t          |          }
t          |	          }|dk    r|
|d         v r	|d         |
= =||d         |
<   I| j        |         dz  dk    r,t          |d         dz            }|                     ||          }t          |d
                   }t          |d                   }t          |d
                                                   }t          |d                                                   }| xj        dz  c_        t          d| j         d|                                            t          d|            t          d| d|dd           t          d| d|dd           dS dS dS dS dS dS # t          $ r}t          d|            Y d}~dS d}~ww xY w)zHandle WebSocket message.streamdata@r   NedepthUpdateubr   ar   
   Ei  r4   z[Snapshot #z] z  ID: z  Bids: z	 levels, z.2fz total volumez  Asks: u      ✗ Error processing message: )jsonloadssplitr   r   r   floatr9   rD   r7   r5   r6   r!   r&   r8   	Exception)r"   wsmessagerG   stream_namer   
depth_datar=   rB   rC   price_floatvolume_floatr/   r!   r@   rA   r>   r?   rI   s                      r   
on_messagez%BinanceMultiAssetCollector.on_message   s3   ;	::g&&D 4FdNN"8n$**3//2--F!&\
 *$$CM)I)I3=c?D(0!%!1&!9J *4C K Kv&+Ell',V}}'1,,*j.@@@$.v$6{$C>JJv.{;; *4C K Kv&+Ell',V}}'1,,*j.@@@$.v$6{$C>JJv.{;; +F3b8A==$'
3$(>$?$?	&*&8&8&K&K%(F);%<%<
%(F);%<%<
+.z&/A/H/H/J/J+K+K(+.z&/A/H/H/J/J+K+K(((A-((PD,<PPPPQQQ4{44555aaa>Naaaabbbaaa>Naaaabbbbbi  NN %$)I)I6 >=   	: 	: 	:8Q88999999999	:s   A
I* HI* *
J4JJc                 *    t          d|            dS )zHandle WebSocket error.u     ✗ WebSocket error: N)r&   )r"   rU   errors      r   on_errorz#BinanceMultiAssetCollector.on_error   s    ///00000r   c                 "   t          d           t          | j                                                  }t          d|            | j                                        D ]-\  }}t          d|                                 d| d           .dS )zHandle WebSocket close.u   
✓ WebSocket closedz  Total snapshots collected: z    z: z
 snapshotsN)r&   r5   r    r6   r<   r8   )r"   rU   close_status_code	close_msgtotal_snapshotsr   counts          r   on_closez#BinanceMultiAssetCollector.on_close   s    '(((d299;;<<?o??@@@!17799 	> 	>MFE<<<5<<<====	> 	>r   c                     t          d           t          dd                    d | j        D                                   t          d           dS )zHandle WebSocket open.u"   ✓ WebSocket connected to Binancez  Symbols: z, c                 6    g | ]}|                                 S r   )r8   r   s     r   r   z6BinanceMultiAssetCollector.on_open.<locals>.<listcomp>   s     &G&G&GQqwwyy&G&G&Gr   u%     ✓ Receiving combined depth streamN)r&   joinr   )r"   rU   s     r   on_openz"BinanceMultiAssetCollector.on_open   sZ    3444JDII&G&G$,&G&G&GHHJJKKK677777r   <   duration_secondsc                    t          d           t          d           t          d           |                                  d                    d | j        D                       }d| }t          d| d           t	          j        || j        | j        | j        | j	                  }t          d	| d
           t          d           	 |
                                 dS # t          $ r t          d           Y dS w xY w)z!Start collecting order book data.zP================================================================================z(BINANCE MULTI-ASSET ORDER BOOK COLLECTOR/c                     g | ]}| d S )z@depthr   r   s     r   r   z4BinanceMultiAssetCollector.start.<locals>.<listcomp>   s    ???QqLLL???r   z-wss://stream.binance.com:9443/stream?streams=z
Connecting to r%   )rh   r[   r^   rd   z
Collecting for z seconds...zPress Ctrl+C to stop early
u   
✓ Collection stopped by userN)r&   r.   rg   r   	websocketWebSocketApprh   r[   r^   rd   run_foreverKeyboardInterrupt)r"   rj   streamsws_urlrU   s        r   startz BinanceMultiAssetCollector.start   s.   f8999f 	 ((??$,???@@JJJ,,,,---#L]]
 
 
 	?"2???@@@,---	7NN  	7 	7 	75666666	7s   C$ $D DN)ri   )__name__
__module____qualname____doc__listr   r#   r.   strr9   rD   r[   r^   rd   rh   rt   r   r   r   r   r      s        HH     +( +( +(Z1C 1C 1C 1 1 1 1f=: =: =:~1 1 1> > >8 8 87 7c 7 7 7 7 7 7r   r   c                  z    g d} d}t          d          }t          | |          }|                    |           dS )zMain collector script.)ethusdtbtcusdtxautusdti,  zT/home/ubuntu/.hermes/workspace/projects/ORDER_FLOW_GRAPH/data/binance_multi_asset.dbN)r   r   rt   )r   durationr	   	collectors       r   mainr      sS      G HkllI +7I>>I OOHr   __main__)rx   rn   rP   r'   pathlibr   r   collectionsr   r:   r   r   ru   r   r   r   <module>r      s                       # # # # # # \7 \7 \7 \7 \7 \7 \7 \7~  & zDFFFFF r   