AlkantarClanX12

Your IP : 18.216.70.205


Current Path : /opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/__pycache__/
Upload File :
Current File : //opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/__pycache__/main.cpython-311.pyc

�

 tf�5��P�ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	ddl
mZmZddl
mZddlmZddlmZmZmZmZmZddlmZmZmZmZddlmZdd	lmZdd
l m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(ddl)m*Z*m+Z+ddl,m-Z-dd
l.m/Z/ddl0m1Z1ed��Z2d�Z3d�Z4eGd�d����Z5d�Z6d%d�Z7d%defd�Z8d%d�Z9d�Z:d�Z;d�Z<d�Z=d�Z>d�Z?d�Z@d �ZAeBd!kr�eCejD����ZEe2�Fd"eE��ee%eE��5ed#ed$��e"��ejejGe3��ejejHe4��ejI��ZJeJ�KeA����ddd��dS#1swxYwYdSdS)&�N)�datetime�	timedelta)�Optional�Iterable)�	dataclass)�or_)�ScrapeResult�DomainAlerts�setup_database�
session_scope�cleanup_old_data)�get_domains�setup_logger�save_pid_and_lock�	intersect)�init_sentry_client)�get_pkg_version)�PING_TIMEOUT_STATUS_CODE�
SENTRY_DNS�!ERROR_DOMAINS_PING_RETRY_INTERVAL�ERROR_DOMAINS_ALERT_INTERVAL�
WMT_LOCK_FILE�PING_CONNECTIONS�&LICENSE_EXPIRED_FAREWELL_LETTER_MARKER�LICENSE_CHECK_PAUSE)�Notifier�SupportedNotificationTypes)�ErrorReport)�cfg)�CloudlinuxLicenseLib�wmt_scannerc��tj��t�dt	tj������dS)NzReloading config: %s)r�reload�logger�info�str�to_dict��
sig_number�frames  ��/builddir/build/BUILDROOT/cl-web-monitoring-tool-3.1.3-1.el8.cloudlinux.x86_64/opt/cloudlinux/venv/lib/python3.11/site-packages/wmt/main.py�reload_confr,-s6���J�L�L�L�
�K�K�&��C�K�M�M�(:�(:�;�;�;�;�;�c�.�tjd��dS)zp
    Shutdown to call finally block to
    close all fds, remove lock and file
    see: save_pid_and_lock()
    rN)�sys�exitr(s  r+�shutdownr12s���H�Q�K�K�K�K�Kr-c�N�eZdZUeed<dZeeed<dZeeed<dS)�ScrapeCoroResult�urlN�
response_code�response_time_ms)	�__name__�
__module__�__qualname__r&�__annotations__r5r�intr6�r-r+r3r3;sD�������	�H�H�H�#'�M�8�C�=�'�'�'�&*��h�s�m�*�*�*�*�*r-r3c��K�tj��4�d{V��}|4�d{V��tj��}	|�||���4�d{V��}t	||jt
dtj��|z
z�����cddd���d{V��cddd���d{V��cddd���d{V��S#1�d{V��swxYwYn�#tjj	$rAt	|t|dz���cYcddd���d{V��cddd���d{V��Stjj$r8t	|d���cYcddd���d{V��cddd���d{V��SwxYw	ddd���d{V��n#1�d{V��swxYwYddd���d{V��dS#1�d{V��swxYwYdS)a
    Main 'pinger'
    1. Requests domains
     - if domain responded - keep status code
     - if no response for timeout - keep Timeout status code
     - if unreachable (ConnectionError or so) - keep 523 status code
       (same logic as go implementation)
    N��timeouti�)r5r6i)r5)
�aiohttp�
ClientSession�time�getr3�statusr;�
concurrent�futures�TimeoutErrorr�client_exceptions�ClientError)r4�ping_timeout�	semaphore�session�start�resps      r+�pingrOCs������$�&�&�<�<�<�<�<�<�<�'�9�<�<�<�<�<�<�<�<��	����	<��{�{�3��{�=�=�
�
�
�
�
�
�
��'��t�{�%(������u�1D�)E�%F�%F����
�
�
�
�
�
�
�
�
�
�
�
�
�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�
�
�
�
�
�
�
�
�
�
����
�
�
�
�
��
�!�.�	J�	J�	J�#�C�2J�5A�D�5H�J�J�J�
J�
J�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<��(�4�	<�	<�	<�$�C�s�;�;�;�;�;�!<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�	<����
�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<����<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<�<����<�<�<�<�<�<s��	F4�F�C�;C	�C�$F4�	
C�C�C�C�F�.E:�	F�
F4�/$E:�F�F4�9E:�:F�>F4�
F	�F4�F	�F4�4
F>�F>c#�K�t��}|�|t|��z}t�dt	|����|D])}tj|��st|||��V��*dS)z�
    ping_timeout: specified in config timeout time (s) for request
    semaphore: semaphore obj to handle asyncio tasks
    ping_target_domains: mostly needed for re-pinging error domains
    Nz Those domains will be pinged: %s)r�setr$�debugr&r�is_domain_ignoredrO)rJrK�ping_target_domains�domains�domains     r+�	executorsrW_s������m�m�G��&��3�*�+�+�+��
�L�L�3�S��\�\�B�B�B��8�8���$�V�,�,�	8��v�|�Y�7�7�7�7�7��8�8r-�returnc���K�d�t|||��D��}t|��dkrgStjt	|��|����d{V��\}}|S)Nc�6�g|]}tj|����Sr<)�asyncio�create_task)�.0�	coroutines  r+�
<listcomp>z scrape_sites.<locals>.<listcomp>os9��
\�
\�
\���
 ��
+�
+�
\�
\�
\r-rr>)rW�lenr[�wait�tuple)�ping_site_timeout�
ping_intervalrKrT�tasks�pinged�_s       r+�scrape_sitesrhns�����
\�
\�'�(9�9�FY�Z�Z�
\�
\�
\�E�
�5�z�z�Q����	��l�5��<�<��G�G�G�G�G�G�G�G�G�I�F�A��Mr-c
��t��}i}g}t|��5}|D]�}|���}|�t	|jd|j|j�����|�|j��|jdkr|j||j<��|�|j����|�	t���tj�
|�����td���d���ddd��n#1swxYwYt!��}	|�|	t|��z}	t|	��|z
}
t|��5}|
D]&}|�t	|d������'	ddd��n#1swxYwY|S)	a
    - obtains all scrape coro results from asyncio tasks
    - saves ping results to ScrapeResult table
    - updates 'is_resolved' field in DomainAlerts table, in case
      error domain`s status code was changed to 200
    - returns domains with non-200 status code code
    T)�website�is_finishedr5r6��)�is_resolvedF)�synchronize_sessionN)rjrk)rQr�result�addr	r4r5r6�append�queryr
�filterrj�in_�update�dictr)�enginerfrT�finished_domains�errors_domains�resolvedrL�taskrorU�unfinished_domains�
unfinisheds            r+�manage_ping_resultsr~ws^���u�u���N��H�	�v�	�	�G�'��	,�	,�D��[�[�]�]�F��K�K���
� �$�2�!'�!8�	���
�
�
�
� � ���,�,�,��#�s�*�*�-3�-A��v�z�*�*�����
�+�+�+�+��
�
�l�#�#�
�V�L�(�,�,�X�6�6�
7�
7�
�V�D�T�*�*�*��V�
F�
F�
F�#G�G�G�G�G�G�G�G�G�G�G����G�G�G�G�$�m�m�G��&��3�*�+�+�+���W���(8�8��	�v�	�	��'�,�	�	�J��K�K��"�!����
�
�
�
�	��������������������s$�DD9�9D=�D=�*F>�>G�Gc	���tj��tt���z
}t	|��5}|�tj���tj�	t|�������ttj
|ktjdk����}d�|D��cddd��S#1swxYwYdS)z�
    - gets websites that must NOT be included in alert email:
          less than ERROR_DOMAINS_ALERT_INTERVAL passed or is_resolved marker was not
          changed from last alerting
    )�hoursFc��g|]	}|j��
Sr<�rj�r]�rows  r+r_z%get_recent_alerts.<locals>.<listcomp>�s��8�8�8����8�8�8r-N)r�nowrrrrrr
rjrsrt�list�keysr�
alert_timerm)rw�
alert_domains�repeat_intervalrL�recently_alerteds     r+�get_recent_alertsr��s���l�n�n�y�7S�'T�'T�'T�T�O�	�v�	�	�9�'�"�=�=��)=�>�>�
�V�L�(�,�,�T�-�2D�2D�2F�2F�-G�-G�H�H���/�/�A�$�0�E�9�;�;�<�<�	�9�8�'7�8�8�8�9�9�9�9�9�9�9�9�9�9�9�9����9�9�9�9�9�9s�B%C+�+C/�2C/c	�Z�t�dtt|���������d�|���D��}t
tjtj	d|itj������|S)z�
    prepares needed error report object with error domains
    to be alerted and sends this mail
    returns alerted domains
    zAlerts will be sent for %sc
��g|]W\}}t|d�ttt	|������t|�������XS)z, )r4�code�count_errors)r�join�mapr&rQr`)r]rV�codess   r+r_zalert.<locals>.<listcomp>�sh�����
�F�E�	�����3�s�C��J�J�/�/�0�0��U���	
�	
�	
���r-�error_report��target_email�
from_email�report�notification_type)
r$r%r&r�r��itemsrrr�r�r�ALERT�notify)�domains_datar�s  r+�alertr��s����K�K�,��D��*�*�,�,�-�-�.�.�0�0�0���*�/�/�1�1�
���L�
��%��>��L�
�5�:�
<�<�<�=C�F�H�H�H��r-c	�`��t||����fd�|���D��}|sOt�dt	t|�������t��dSt|��tj
��}t|��5}|�t���tj�����}d�|D��}|D]�}||vr^|�t���tj|k���t'|d������d|�t||�������	ddd��dS#1swxYwYdS)aJ
    - gets recently alerted domains (those that must not be alerted again)
      and does not include them for alerting
    - calls alerting for left domains
    - updates DomainAlerts table:
      if website was not alerted -> adds new record
      if website was alerted before -> updates alert time and is_resolved marker
    c�$��i|]\}}|�v�	||��
Sr<r<)r]�k�vr�s   �r+�
<dictcomp>z flush_alerts.<locals>.<dictcomp>�s*���\�\�\���A�!�K[�B[�B[��1�B[�B[�B[r-zDAll domains "%s" were alerted or still not resolved in last %d hoursNc��g|]	}|j��
Sr<r�r�s  r+r_z flush_alerts.<locals>.<listcomp>�s��0�0�0����0�0�0r-F)r�rm)rjr�)r�r�r$r%r&r�r�rr�rr�rrrr
�
with_entitiesrj�allrsrurvrp)	rwr��domains_to_alertr�rL�websites�urlsrVr�s	        @r+�flush_alertsr��s����)���?�?��\�\�\�\��)<�)<�)>�)>�\�\�\�������Z���]�/�/�1�1�2�2�3�3�0�	2�	2�	2�	��	�
����
�,�.�.�C�	�v�	�	�J�'��=�=��.�.�
�]�<�/�
0�
0�
�S�U�U�	�1�0�x�0�0�0��&�	J�	J�F���~�~��
�
�l�+�+��V�L�0�F�:�;�;��V�D�C�U�C�C�C�D�D�D�D����L��C�H�H�H�I�I�I�I�
	J�J�J�J�J�J�J�J�J�J�J�J�J����J�J�J�J�J�Js�4C!F#�#F'�*F'c�0�|rtjjrdSdS)NTF)r�alert_notifications_enabled)�
error_domainss r+�should_be_repingedr��s ������<���t��5r-c��tj�t��r5t�d��tjt��dSdS)NzCloudLinux license was updated)�os�path�existsrr$r%�remover<r-r+�cleanup_farewell_letter_markerr��sM��	�w�~�~�<�=�=�:����4�5�5�5�
�	�8�9�9�9�9�9�:�:r-c�d�	tj�t��s_t�d��t
tjtj	itj������dSdS#t$rt�d��YdSwxYw)z@
    Sends farewell letter once (if it was not sent before)
    z/Going to send last email about expired license!r�z$Error while managing farewell letterN)r�r�r�rr$�warningrrr�r�r�FAREWELLr��	Exception�	exceptionr<r-r+�manage_license_farewellr��s���	A��w�~�~�D�E�E�	P��N�N�L�M�M�M�� �-��>��"<�"E�	
G�
G�
G�HN�v�x�x�x�x�x�
	P�	P���A�A�A����?�@�@�@�@�@�@�A���s�BB�$B/�.B/c	��zK�tj��}tjjdz}	t��t	|��tjt��}ttjj	||���d{V��}t||��}t|���r�t�
dt|����	tjjdz}|tkr�tj��|z
}tjt#t%t|z
d��d�����d{V��|tkrnttjj	|||�������d{V��}t|||�����}	t)||	��}
nnt)||��}
|}n��|
rWt�
dtt+|
���������t-||
��ng}n*#t.$rt�d	��YnwxYw	tjjdz}tj��|z
}t#t%||z
d��d��}tj|���d{V��||krn�mns#	tjjdz}tj��|z
}t#t%||z
d��d��}tj|���d{V��||krn�mwxYw|S)
aF
    Scanner logic:
    1. Scrapes domains and obtains ping results;
    2. Manage ping results (e.g: saving to DB)
    3. In case error domains found -> start re-pinging
       Re-pinging:
         - in min(ping_interval, 5 mins)
         - flush alerts if needed
    4. Sleep for ping_interval until next ping iteration
    �<Nz8Those domains are unsuccessful: %s 
 Try to re-ping themTr�
)rTz1Domains with unsuccessful status code found: "%s"zError during ping iteration!)rBrrdr�r
r[�	SemaphorerrhrJr~r�r$r%r&r�sleep�min�maxr�rr�r�r�r�)
�previously_erroredrwrM�ping_interval_secondsrK�ping_resultr��elapsed_for_ping�ping_retry_result�retry_errorsr��elapsed�
sleep_times
             r+�scrape_iterationr�	s�����
�I�K�K�E��G�1�B�6��2�&�(�(�(��� � � ��%�&6�7�7�	�(���)=�?T�V_�`�`�`�`�`�`�`�`��+�F�K�@�@�
��m�,�,� 	$��K�K�S�UX�Yf�Ug�Ug�h�h�h�
�(+��(=��(B�%�(�+L�L�L�'+�y�{�{�U�':�$�!�-��C� A�DT� T�VW�X�X�Z\�]�]����������(�*K�K�K�2>��G�0�2G��0=�0B�0B�0D�0D�3�3�3�-�-�-�-�-�-�)�(;�6�CT�Vc�Vh�Vh�Vj�Vj�'k�'k��(1�-��(N�(N�
��L�%.�.@�-�$P�$P�M�)6�&��+
�.�
4����O���]�%7�%7�%9�%9� :� :�;�;�=�=�=��V�]�3�3�3��"$�����9�9�9����7�8�8�8�8�8�9����	�$'�G�$9�B�$>�!��i�k�k�E�)�G��S�!6��!@�!�D�D�b�I�I�J��-�
�+�+�+�+�+�+�+�+�+��.�.�.��
	���
	�$'�G�$9�B�$>�!��i�k�k�E�)�G��S�!6��!@�!�D�D�b�I�I�J��-�
�+�+�+�+�+�+�+�+�+��.�.�.��
	�	�	�	�	��s+�HH/�.K�/$I�K�I�K�A0L8c��K�t��}g}d}	t�����rd}t||���d{V��}n~|dz
}|dkrt�d��|dkrt
��d}tjjdz}t|t��}tj|���d{V����)z�
    Main loop for wmt_scanner_solo service
    each 'while: True' iteration returns errored domains
    (domains that responded with non-200 status code)
    rTN�z)Seems your CloudLinux license is expired!�r�)
rr �get_license_statusr�r$r�r�rrdr�rr[r�)rwr��license_attemptr�r�s     r+�scrape_loopr�Ms������
�
�F����O�,��!�!�4�4�6�6�	,��O�'7�8J�F�'S�'S�!S�!S�!S�!S�!S�!S����q� �O��!�#�#����J�K�K�K��!�#�#�'�)�)�)�"#��$'�G�$9�B�$>�!��2�4G�H�H�J��-�
�+�+�+�+�+�+�+�+�+�,r-�__main__zPID: %szweb-monitoring-toolzcl-web-monitoring-tool)N)Lr[r�r/r@�signalrB�concurrent.futuresrErr�typingrr�dataclassesr�
sqlalchemyr�wmt.dbr	r
rrr
�wmt.common.utilsrrrr�clsentryr�clsentry.utilsr�wmt.common.constrrrrrrrr�wmt.common.notificationrr�wmt.common.reportr�
wmt.commonr�	cllicenser r$r,r1r3rOrWrhr~r�r�r�r�r�r�r�r�r7r&�getpid�pidr%�SIGUSR1�SIGTERM�get_event_loop�loop�run_until_completer<r-r+�<module>r�sX������	�	�	�	�
�
�
�
�����
�
�
�
���������(�(�(�(�(�(�(�(�%�%�%�%�%�%�%�%�!�!�!�!�!�!���������������������U�T�T�T�T�T�T�T�T�T�T�T�'�'�'�'�'�'�*�*�*�*�*�*�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�I�H�H�H�H�H�H�H�)�)�)�)�)�)�������*�*�*�*�*�*�	��m�	$�	$��<�<�<�
����+�+�+�+�+�+�+���+�<�<�<�88�8�8�8���ai�����(�(�(�(�V9�9�9����4J�J�J�D���:�:�:�
A�
A�
A� A�A�A�H,�,�,�6�z���

�#�i�b�i�k�k�
�
�C�
�K�K�	�3����	�	�=�#�	.�	.�/�/���0�*�?�+C�D�D�%�	'�	'�	'�	��
�f�n�k�2�2�2���
�f�n�h�/�/�/�%�w�%�'�'�������
�
�.�.�.�/�/�/�/�/�/�/�/�/�/�/�/����/�/�/�/�/�/��s�A/F�F� F