This commit is contained in:
lifestorm
2024-08-04 22:55:00 +03:00
parent 0e770b2b49
commit 94063e4369
7342 changed files with 1718932 additions and 14 deletions

View File

@@ -0,0 +1,63 @@
--[[
| This file was obtained through the combined efforts
| of Madbluntz & Plymouth Antiquarian Society.
|
| Credits: lifestorm, Gregory Wayne Rossel JR.,
| Maloy, DrPepper10 @ RIP, Atle!
|
| Visit for more: https://plymouth.thetwilightzone.ru/
--]]
express._receiverMadeQueue = {}
express._canSendReceiverMade = false
net.Receive( "express_access", function()
express:SetAccess( net.ReadString() )
express:_sendReceiversMadeQueue()
end )
function express:_sendReceiversMadeQueue()
express._canSendReceiverMade = true
local messages = table.GetKeys( express._receiverMadeQueue )
express:_alertReceiversMade( unpack( messages ) )
end
function express:_alertReceiversMade( ... )
local names = { ... }
local receiverCount = #names
net.Start( "express_receivers_made" )
net.WriteUInt( receiverCount, 8 )
for i = 1, receiverCount do
net.WriteString( names[i] )
end
net.SendToServer()
end
-- Registers a basic receiver --
function express.Receive( message, cb )
express:_setReceiver( message, cb )
if not express._canSendReceiverMade then
express._receiverMadeQueue[message] = true
return
end
express:_alertReceiversMade( message )
end
-- Calls the main _send function but passes nil for the recipient --
function express.Send( message, data, onProof )
express:_send( message, data, nil, onProof )
end
function express:SetExpected( hash, cb )
self._awaitingProof[hash] = cb
end

View File

@@ -0,0 +1,278 @@
--[[
| This file was obtained through the combined efforts
| of Madbluntz & Plymouth Antiquarian Society.
|
| Credits: lifestorm, Gregory Wayne Rossel JR.,
| Maloy, DrPepper10 @ RIP, Atle!
|
| Visit for more: https://plymouth.thetwilightzone.ru/
--]]
AddCSLuaFile()
express.version = 1
express.revision = 1
express._putCache = {}
express._maxCacheTime = ( 24 - 1 ) * 60 * 60 -- TODO: Get this from the server, similar to the version check
express._waitingForAccess = {}
express.domain = CreateConVar(
"express_domain", "gmod.express", FCVAR_ARCHIVE + FCVAR_REPLICATED, "The domain of the Express server"
)
-- Useful for self-hosting if you need to set express_domain to localhost
-- and direct clients to a global IP/domain to hit the same service
express.domain_cl = CreateConVar(
"express_domain_cl", "", FCVAR_ARCHIVE + FCVAR_REPLICATED, "The client-specific domain of the Express server. If empty, express_domain will be used."
)
express.sendDelay = CreateConVar(
"express_send_delay", 0.15, FCVAR_ARCHIVE + FCVAR_REPLICATED, "How long to wait (in seconds) before sending the Express Message ID to the recipient (longer delays will result in increased reliability)"
)
-- Runs the correct net Send function based on the realm --
function express.shSend( target )
if CLIENT then
net.SendToServer()
else
net.Send( target )
end
end
-- Returns the correct domain based on the realm and convars --
function express:getDomain()
local domain = self.domain:GetString()
if SERVER then return domain end
local clDomain = self.domain_cl:GetString()
if clDomain ~= "" then return clDomain end
return domain
end
-- Creates the base of the API URL from the protocol, domain, and version --
function express:makeBaseURL()
local protocol = self._protocol
local domain = self:getDomain()
return string.format( "%s://%s/v%d", protocol, domain, self.version )
end
-- Creates a full URL with the given access token --
function express:makeAccessURL( action, ... )
local url = self:makeBaseURL()
local args = { action, self.access, ... }
return url .. "/" .. table.concat( args, "/" )
end
-- Sets the access token and runs requests that were waiting --
function express:SetAccess( access )
self.access = access
local waiting = self._waitingForAccess
for _, callback in ipairs( waiting ) do
callback()
end
self._waitingForAccess = {}
end
-- Checks the version of the API and alerts of a mismatch --
function express.CheckRevision()
local suffix = " on version check! This is bad!"
local err = function( msg )
return "Express: " .. msg .. suffix
end
local url = express:makeBaseURL() .. "/revision"
local success = function( body, _, _, code )
assert( code >= 200 and code < 300, err( "Invalid response code (" .. code .. ")" ) )
local dataHolder = util.JSONToTable( body )
assert( dataHolder, err( "Invalid JSON response" ) )
local revision = dataHolder.revision
assert( revision, err( "Invalid JSON response" ) )
local current = express.revision
if revision ~= current then
error( err( "Revision mismatch! Expected " .. current .. ", got " .. revision ) )
end
end
http.Fetch( url, success, function( message )
error( err( message ) )
end, express.jsonHeaders )
end
-- Runs the main :Get function, or queues the request if no access token is set --
function express:_get( id, cb )
if self.access then
return self:Get( id, cb )
end
table.insert( self._waitingForAccess, function()
self:Get( id, cb )
end )
end
-- Runs the main :GetSize function, or queues the request if no access token is set --
-- FIXME: If this gets delayed because it doesn't have an access token, the PreDl Receiver will not be able to stop the download --
function express:_getSize( id, cb )
if self.access then
return self:GetSize( id, cb )
end
table.insert( self._waitingForAccess, function()
self:GetSize( id, cb )
end )
end
---Encodes and compresses the given data, then sends it to the API if not already cached
function express:_put( data, cb )
if table.Count( data ) == 0 then
error( "Express: Tried to send empty data!" )
end
data = pon.encode( data )
if string.len( data ) > self._maxDataSize then
data = "<enc>" .. util.Compress( data )
assert( data, "Express: Failed to compress data!" )
local dataLen = string.len( data )
if dataLen > self._maxDataSize then
error( "Express: Data too large (" .. dataLen .. " bytes)" )
end
end
local hash = util.SHA1( data )
local cached = self._putCache[hash]
if cached then
local cachedAt = cached.cachedAt
if os.time() <= ( cachedAt + self._maxCacheTime ) then
-- Force the callback to run asynchronously for consistency
timer.Simple( 0, function()
cb( cached.id, hash )
end )
return
end
end
local function wrapCb( id )
self._putCache[hash] = { id = id, cachedAt = os.time() }
cb( id, hash )
end
if self.access then
return self:Put( data, wrapCb )
end
table.insert( self._waitingForAccess, function()
self:Put( data, wrapCb )
end )
end
-- TODO: Fix GLuaTest so we can actually test this function...
-- Creates a contextual callback for the :_put endpoint, delaying the notification to the recipient(s) --
function express:_putCallback( message, plys, onProof )
return function( id, hash )
if onProof then
self:SetExpected( hash, onProof, plys )
end
-- Cloudflare isn't fulfilling their promise that the first lookup in
-- each region will "search" for the target key in K/V if it has't been cached yet.
-- This delay makes it more likely that the data will have "settled" into K/V before the first lookup
-- (Once it's cached as a 404, it'll stay that way for about 60 seconds)
timer.Simple( self.sendDelay:GetFloat(), function()
net.Start( "express" )
net.WriteString( message )
net.WriteString( id )
net.WriteBool( onProof ~= nil )
express.shSend( plys )
end )
end
end
-- Calls the _put function with a contextual callback --
function express:_send( message, data, plys, onProof )
self:_put( data, self:_putCallback( message, plys, onProof ) )
end
-- Assigns a callback to the given message --
function express:_setReceiver( message, cb )
message = string.lower( message )
self._receivers[message] = cb
end
-- Returns the receiver set for the given message --
function express:_getReceiver( message )
message = string.lower( message )
return self._receivers[message]
end
-- Returns the pre-download receiver set for the given message --
function express:_getPreDlReceiver( message )
message = string.lower( message )
return self._preDlReceivers[message]
end
-- Returns a realm-specific timeout value for HTTP requests --
function express:_getTimeout()
return CLIENT and 240 or 60
end
-- Ensures that the given HTTP response code indicates a succcessful request --
function express._checkResponseCode( code )
local isOk = isnumber( code ) and code >= 200 and code < 300
if isOk then return end
error( "Express: Invalid response code (" .. tostring( code ) .. ")" )
end
-- Attempts to re-register with the new domain, and then verifies its version --
cvars.AddChangeCallback( "express_domain", function()
express._putCache = {}
if SERVER then express:Register() end
express:CheckRevision()
end, "domain_check" )
-- Both client and server should check the version on startup so that errors are caught early --
cvars.AddChangeCallback( "express_domain_cl", function( _, _, new )
if CLIENT then express._putCache = {} end
if new == "" then return end
express:CheckRevision()
end, "domain_check" )
hook.Add( "ExpressLoaded", "Express_HTTPInit", function()
hook.Add( "Tick", "Express_RevisionCheck", function()
hook.Remove( "Tick", "Express_RevisionCheck" )
if SERVER then express:Register() end
express:CheckRevision()
end )
end )

227
lua/gm_express/sh_init.lua Normal file
View File

@@ -0,0 +1,227 @@
--[[
| This file was obtained through the combined efforts
| of Madbluntz & Plymouth Antiquarian Society.
|
| Credits: lifestorm, Gregory Wayne Rossel JR.,
| Maloy, DrPepper10 @ RIP, Atle!
|
| Visit for more: https://plymouth.thetwilightzone.ru/
--]]
AddCSLuaFile()
require( "pon" )
if SERVER then
util.AddNetworkString( "express" )
util.AddNetworkString( "express_proof" )
util.AddNetworkString( "express_receivers_made" )
end
express = {}
express._receivers = {}
express._protocol = "http"
express._awaitingProof = {}
express._preDlReceivers = {}
express._maxDataSize = 24 * 1024 * 1024
express._jsonHeaders = { ["Content-Type"] = "application/json" }
express._bytesHeaders = { ["Accept"] = "application/octet-stream" }
-- Removes a receiver --
function express.ClearReceiver( message )
message = string.lower( message )
express._receivers[message] = nil
end
-- Registers a PreDownload receiver --
function express.ReceivePreDl( message, preDl )
message = string.lower( message )
express._preDlReceivers[message] = preDl
end
-- Retrieves and parses the data for given ID --
function express:Get( id, cb, _attempts )
_attempts = _attempts or 0
local url = self:makeAccessURL( "read", id )
local success = function( code, body )
if code == 404 then
assert( _attempts <= 35, "express:Get() failed to retrieve data after 35 attempts: " .. id )
timer.Simple( 0.125 * _attempts, function()
self:Get( id, cb, _attempts + 1 )
end )
return
end
express._checkResponseCode( code )
if _attempts > 0 then
print( "express:Get() succeeded after " .. _attempts .. " attempts: " .. id )
end
if string.StartWith( body, "<enc>" ) then
body = util.Decompress( string.sub( body, 6 ) )
if ( not body ) or #body == 0 then
error( "Express: Failed to decompress data for ID '" .. id .. "'." )
end
end
local hash = util.SHA1( body )
local decodedData = pon.decode( body )
cb( decodedData, hash )
end
HTTP( {
method = "GET",
url = url,
success = success,
failed = error,
headers = self._bytesHeaders,
timeout = self:_getTimeout()
} )
end
-- Asks the API for this ID's data's size --
function express:GetSize( id, cb )
local url = self:makeAccessURL( "size", id )
local success = function( code, body )
express._checkResponseCode( code )
local sizeHolder = util.JSONToTable( body )
assert( sizeHolder, "Invalid JSON" )
local size = sizeHolder.size
if not size then
print( "Express: Failed to get size for ID '" .. id .. "'.", code )
print( body )
end
assert( size, "No size data" )
cb( tonumber( size ) )
end
HTTP( {
method = "GET",
url = url,
success = success,
failed = error,
headers = self._jsonHeaders,
timeout = self:_getTimeout()
} )
end
-- Given prepared data, sends it to the API --
function express:Put( data, cb )
local success = function( code, body )
express._checkResponseCode( code )
local response = util.JSONToTable( body )
assert( response, "Invalid JSON" )
assert( response.id, "No ID returned" )
cb( response.id )
end
HTTP( {
method = "POST",
url = self:makeAccessURL( "write" ),
body = data,
success = success,
failed = error,
headers = {
["Content-Length"] = #data,
["Accept"] = "application/json"
},
type = "application/octet-stream",
timeout = CLIENT and 240 or 60
} )
end
-- Runs the express receiver for the given message --
function express:Call( message, ply, data )
local cb = self:_getReceiver( message )
if not cb then return end
if CLIENT then return cb( data ) end
if SERVER then return cb( ply, data ) end
end
-- Runs the express pre-download receiver for the given message --
function express:CallPreDownload( message, ply, id, size, needsProof )
local cb = self:_getPreDlReceiver( message )
if not cb then return end
if CLIENT then return cb( message, id, size, needsProof ) end
if SERVER then return cb( message, ply, id, size, needsProof ) end
end
-- Handles a net message containing an ID to download from the API --
function express.OnMessage( _, ply )
local message = net.ReadString()
if not express:_getReceiver( message ) then
error( "Express: Received a message that has no listener! (" .. message .. ")" )
end
local id = net.ReadString()
local needsProof = net.ReadBool()
local function makeRequest( size )
if size then
local check = express:CallPreDownload( message, ply, id, size, needsProof )
if check == false then return end
end
express:_get( id, function( data, hash )
express:Call( message, ply, data )
if not needsProof then return end
net.Start( "express_proof" )
net.WriteString( hash )
express.shSend( ply )
end )
end
if express:_getPreDlReceiver( message ) then
return express:_getSize( id, makeRequest )
end
makeRequest()
end
-- Handles a net message containing a proof of data download --
function express.OnProof( _, ply )
-- Server prefixes the hash with the player's Steam ID
local prefix = ply and ply:SteamID64() .. "-" or ""
local hash = prefix .. net.ReadString()
local cb = express._awaitingProof[hash]
if not cb then return end
cb( ply )
express._awaitingProof[hash] = nil
end
net.Receive( "express", express.OnMessage )
net.Receive( "express_proof", express.OnProof )
include( "sh_helpers.lua" )
if SERVER then
include( "sv_init.lua" )
AddCSLuaFile( "cl_init.lua" )
else
include( "cl_init.lua" )
end
hook.Add( "CreateTeams", "ExpressLoaded", function()
hook.Run( "ExpressLoaded" )
end )

View File

@@ -0,0 +1,94 @@
--[[
| This file was obtained through the combined efforts
| of Madbluntz & Plymouth Antiquarian Society.
|
| Credits: lifestorm, Gregory Wayne Rossel JR.,
| Maloy, DrPepper10 @ RIP, Atle!
|
| Visit for more: https://plymouth.thetwilightzone.ru/
--]]
require( "playerload" )
util.AddNetworkString( "express_access" )
-- Registers a basic receiver --
function express.Receive( message, cb )
express:_setReceiver( message, cb )
end
-- Broadcasts the given data to all connected players --
function express.Broadcast( message, data, onProof )
express.Send( message, data, player.GetAll(), onProof )
end
-- Registers with the current API, storing and distributing access tokens --
function express.Register()
-- All stored items expire after a day
-- That includes tokens, so we need
-- to re-register if we make it this far
local oneDay = 60 * 60 * 24
timer.Create( "Express_Register", oneDay, 0, express.Register )
local url = express:makeBaseURL() .. "/register"
http.Fetch( url, function( body, _, _, code )
express._checkResponseCode( code )
local response = util.JSONToTable( body )
assert( response, "Invalid JSON" )
assert( response.server, "No server access token" )
assert( response.client, "No client access token" )
express:SetAccess( response.server )
express._clientAccess = response.client
if player.GetCount() == 0 then return end
net.Start( "express_access" )
net.WriteString( express._clientAccess )
net.Broadcast()
end, error, express.jsonHeaders )
end
-- Passthrough for the shared _send function --
function express.Send( ... )
express:_send( ... )
end
-- Sets a callback for each of the recipients that will run when they provide proof --
function express:SetExpected( hash, cb, plys )
if not istable( plys ) then plys = { plys } end
for _, ply in ipairs( plys ) do
local key = ply:SteamID64() .. "-" .. hash
self._awaitingProof[key] = cb
end
end
-- Runs a hook when a player makes a new express Receiver --
function express._onReceiverMade( _, ply )
local messageCount = net.ReadUInt( 8 )
for _ = 1, messageCount do
local name = string.lower( net.ReadString() )
hook.Run( "ExpressPlayerReceiver", ply, name )
end
end
net.Receive( "express_receivers_made", express._onReceiverMade )
-- Send the player their access token as soon as it's safe to do so --
function express._onPlayerLoaded( ply )
net.Start( "express_access" )
net.WriteString( express._clientAccess )
net.Send( ply )
end
hook.Add( "PlayerFullLoad", "Express_PlayerReady", express._onPlayerLoaded )