You cannot select more than 25 topics
			Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
		
		
		
		
		
			
		
			
				
	
	
		
			183 lines
		
	
	
		
			7.2 KiB
		
	
	
	
		
			Python
		
	
			
		
		
	
	
			183 lines
		
	
	
		
			7.2 KiB
		
	
	
	
		
			Python
		
	
"""Gateway connection classes."""
 | 
						|
 | 
						|
# Copyright (c) Jupyter Development Team.
 | 
						|
# Distributed under the terms of the Modified BSD License.
 | 
						|
from __future__ import annotations
 | 
						|
 | 
						|
import asyncio
 | 
						|
import logging
 | 
						|
import random
 | 
						|
from typing import Any, cast
 | 
						|
 | 
						|
import tornado.websocket as tornado_websocket
 | 
						|
from tornado.concurrent import Future
 | 
						|
from tornado.escape import json_decode, url_escape, utf8
 | 
						|
from tornado.httpclient import HTTPRequest
 | 
						|
from tornado.ioloop import IOLoop
 | 
						|
from traitlets import Bool, Instance, Int, Unicode
 | 
						|
 | 
						|
from ..services.kernels.connection.base import BaseKernelWebsocketConnection
 | 
						|
from ..utils import url_path_join
 | 
						|
from .gateway_client import GatewayClient
 | 
						|
 | 
						|
 | 
						|
class GatewayWebSocketConnection(BaseKernelWebsocketConnection):
 | 
						|
    """Web socket connection that proxies to a kernel/enterprise gateway."""
 | 
						|
 | 
						|
    ws = Instance(klass=tornado_websocket.WebSocketClientConnection, allow_none=True)
 | 
						|
 | 
						|
    ws_future = Instance(klass=Future, allow_none=True)
 | 
						|
 | 
						|
    disconnected = Bool(False)
 | 
						|
 | 
						|
    retry = Int(0)
 | 
						|
 | 
						|
    # When opening ws connection to gateway, server already negotiated subprotocol with notebook client.
 | 
						|
    # Same protocol must be used for client and gateway, so legacy ws subprotocol for client is enforced here.
 | 
						|
 | 
						|
    kernel_ws_protocol = Unicode("", allow_none=True, config=True)
 | 
						|
 | 
						|
    async def connect(self):
 | 
						|
        """Connect to the socket."""
 | 
						|
        # websocket is initialized before connection
 | 
						|
        self.ws = None
 | 
						|
        ws_url = url_path_join(
 | 
						|
            GatewayClient.instance().ws_url or "",
 | 
						|
            GatewayClient.instance().kernels_endpoint,
 | 
						|
            url_escape(self.kernel_id),
 | 
						|
            "channels",
 | 
						|
        )
 | 
						|
        if self.session_id:
 | 
						|
            ws_url += f"?session_id={url_escape(self.session_id)}"
 | 
						|
        self.log.info(f"Connecting to {ws_url}")
 | 
						|
        kwargs: dict[str, Any] = {}
 | 
						|
        kwargs = GatewayClient.instance().load_connection_args(**kwargs)
 | 
						|
 | 
						|
        request = HTTPRequest(ws_url, **kwargs)
 | 
						|
        self.ws_future = cast("Future[Any]", tornado_websocket.websocket_connect(request))
 | 
						|
        self.ws_future.add_done_callback(self._connection_done)
 | 
						|
 | 
						|
        loop = IOLoop.current()
 | 
						|
        loop.add_future(self.ws_future, lambda future: self._read_messages())
 | 
						|
 | 
						|
    def _connection_done(self, fut):
 | 
						|
        """Handle a finished connection."""
 | 
						|
        if (
 | 
						|
            not self.disconnected and fut.exception() is None
 | 
						|
        ):  # prevent concurrent.futures._base.CancelledError
 | 
						|
            self.ws = fut.result()
 | 
						|
            self.retry = 0
 | 
						|
            self.log.debug(f"Connection is ready: ws: {self.ws}")
 | 
						|
        else:
 | 
						|
            self.log.warning(
 | 
						|
                "Websocket connection has been closed via client disconnect or due to error.  "
 | 
						|
                f"Kernel with ID '{self.kernel_id}' may not be terminated on GatewayClient: {GatewayClient.instance().url}"
 | 
						|
            )
 | 
						|
 | 
						|
    def disconnect(self):
 | 
						|
        """Handle a disconnect."""
 | 
						|
        self.disconnected = True
 | 
						|
        if self.ws is not None:
 | 
						|
            # Close connection
 | 
						|
            self.ws.close()
 | 
						|
        elif self.ws_future and not self.ws_future.done():
 | 
						|
            # Cancel pending connection.  Since future.cancel() is a noop on tornado, we'll track cancellation locally
 | 
						|
            self.ws_future.cancel()
 | 
						|
            self.log.debug(f"_disconnect: future cancelled, disconnected: {self.disconnected}")
 | 
						|
 | 
						|
    async def _read_messages(self):
 | 
						|
        """Read messages from gateway server."""
 | 
						|
        while self.ws is not None:
 | 
						|
            message = None
 | 
						|
            if not self.disconnected:
 | 
						|
                try:
 | 
						|
                    message = await self.ws.read_message()
 | 
						|
                except Exception as e:
 | 
						|
                    self.log.error(
 | 
						|
                        f"Exception reading message from websocket: {e}"
 | 
						|
                    )  # , exc_info=True)
 | 
						|
                if message is None:
 | 
						|
                    if not self.disconnected:
 | 
						|
                        self.log.warning(f"Lost connection to Gateway: {self.kernel_id}")
 | 
						|
                    break
 | 
						|
                if isinstance(message, bytes):
 | 
						|
                    message = message.decode("utf8")
 | 
						|
                self.handle_outgoing_message(
 | 
						|
                    message
 | 
						|
                )  # pass back to notebook client (see self.on_open and WebSocketChannelsHandler.open)
 | 
						|
            else:  # ws cancelled - stop reading
 | 
						|
                break
 | 
						|
 | 
						|
        # NOTE(esevan): if websocket is not disconnected by client, try to reconnect.
 | 
						|
        if not self.disconnected and self.retry < GatewayClient.instance().gateway_retry_max:
 | 
						|
            jitter = random.randint(10, 100) * 0.01  # noqa: S311
 | 
						|
            retry_interval = (
 | 
						|
                min(
 | 
						|
                    GatewayClient.instance().gateway_retry_interval * (2**self.retry),
 | 
						|
                    GatewayClient.instance().gateway_retry_interval_max,
 | 
						|
                )
 | 
						|
                + jitter
 | 
						|
            )
 | 
						|
            self.retry += 1
 | 
						|
            self.log.info(
 | 
						|
                "Attempting to re-establish the connection to Gateway in %s secs (%s/%s): %s",
 | 
						|
                retry_interval,
 | 
						|
                self.retry,
 | 
						|
                GatewayClient.instance().gateway_retry_max,
 | 
						|
                self.kernel_id,
 | 
						|
            )
 | 
						|
            await asyncio.sleep(retry_interval)
 | 
						|
            loop = IOLoop.current()
 | 
						|
            loop.spawn_callback(self.connect)
 | 
						|
 | 
						|
    def handle_outgoing_message(self, incoming_msg: str, *args: Any) -> None:
 | 
						|
        """Send message to the notebook client."""
 | 
						|
        try:
 | 
						|
            self.websocket_handler.write_message(incoming_msg)
 | 
						|
        except tornado_websocket.WebSocketClosedError:
 | 
						|
            if self.log.isEnabledFor(logging.DEBUG):
 | 
						|
                msg_summary = GatewayWebSocketConnection._get_message_summary(
 | 
						|
                    json_decode(utf8(incoming_msg))
 | 
						|
                )
 | 
						|
                self.log.debug(
 | 
						|
                    f"Notebook client closed websocket connection - message dropped: {msg_summary}"
 | 
						|
                )
 | 
						|
 | 
						|
    def handle_incoming_message(self, message: str) -> None:
 | 
						|
        """Send message to gateway server."""
 | 
						|
        if self.ws is None and self.ws_future is not None:
 | 
						|
            loop = IOLoop.current()
 | 
						|
            loop.add_future(self.ws_future, lambda future: self.handle_incoming_message(message))
 | 
						|
        else:
 | 
						|
            self._write_message(message)
 | 
						|
 | 
						|
    def _write_message(self, message):
 | 
						|
        """Send message to gateway server."""
 | 
						|
        try:
 | 
						|
            if not self.disconnected and self.ws is not None:
 | 
						|
                self.ws.write_message(message)
 | 
						|
        except Exception as e:
 | 
						|
            self.log.error(f"Exception writing message to websocket: {e}")  # , exc_info=True)
 | 
						|
 | 
						|
    @staticmethod
 | 
						|
    def _get_message_summary(message):
 | 
						|
        """Get a summary of a message."""
 | 
						|
        summary = []
 | 
						|
        message_type = message["msg_type"]
 | 
						|
        summary.append(f"type: {message_type}")
 | 
						|
 | 
						|
        if message_type == "status":
 | 
						|
            summary.append(", state: {}".format(message["content"]["execution_state"]))
 | 
						|
        elif message_type == "error":
 | 
						|
            summary.append(
 | 
						|
                ", {}:{}:{}".format(
 | 
						|
                    message["content"]["ename"],
 | 
						|
                    message["content"]["evalue"],
 | 
						|
                    message["content"]["traceback"],
 | 
						|
                )
 | 
						|
            )
 | 
						|
        else:
 | 
						|
            summary.append(", ...")  # don't display potentially sensitive data
 | 
						|
 | 
						|
            return "".join(summary)
 |