当前位置： 首页 > article >正文

Laravel对接SLS日志服务

article 2024/11/26 2:16:53

Laravel对接SLS日志服务（写入和读取）

1、下载阿里云的sdk

#通过composer下载
composer require  alibabacloud/aliyun-log-php-sdk

#对应的git仓库 
https://github.com/aliyun/aliyun-log-php-sdk

2、创建sdk请求的service

<?php

namespace App\Services;

use Aliyun_Log_Client;
use Aliyun_Log_Models_LogItem;
use Aliyun_Log_Models_PutLogsRequest;
use Aliyun_Log_Models_GetLogsRequest;
use Aliyun_Log_Models_GetHistogramsRequest;
use Exception;
use Illuminate\Support\Facades\Log;

class SLSTimeSeriesService
{
    protected $client;
    protected $project;
    protected $logstore;

    public function __construct()
    {
        // 从配置中获取凭证
        $endpoint = env('ALIYUN_SLS_ENDPOINT');
        $accessKeyId = env('ALIYUN_ACCESS_KEY_ID');
        $accessKeySecret = env('ALIYUN_ACCESS_KEY_SECRET');

        // 验证配置
        if (!$endpoint || !$accessKeyId || !$accessKeySecret) {
            throw new Exception('SLS configuration is missing');
        }

        // 初始化客户端
        $this->client = new Aliyun_Log_Client($endpoint, $accessKeyId, $accessKeySecret);
        $this->project = env('ALIYUN_SLS_PROJECT');
        $this->logstore = env('ALIYUN_SLS_LOGSTORE');
    }


    /**
     * 查询日志（带分页和排序）
     */
    public function getLogs($query = '', $from = null, $to = null, $page = 1, $perPage = 10, $sort = 'desc', $sortField = '__time__')
    {
        try {
            $from = $from ?: time() - 3600;
            $to = $to ?: time();

            // 计算偏移量
            $offset = ($page - 1) * $perPage;

            // 获取总数
            $histogramRequest = new Aliyun_Log_Models_GetHistogramsRequest(
                $this->project,
                $this->logstore,
                $from,
                $to,
                '',
                $query ?: '*'
            );

            $histogramResponse = $this->client->getHistograms($histogramRequest);
            $total = $histogramResponse->getTotalCount();

            // 创建日志查询请求
            $request = new Aliyun_Log_Models_GetLogsRequest(
                $this->project,
                $this->logstore,
                $from,
                $to,
                '',
                $query ?: '*',
                1000,  // 先获取较多数据以便排序
                0,
                true
            );

            // 执行查询
            $response = $this->client->getLogs($request);
            $logs = $response->getLogs();

            // 处理日志数据
            $formattedLogs = [];
            foreach ($logs as $log) {
                $contents = $log->getContents();
                $contents['log_time'] = date('Y-m-d H:i:s', $log->getTime());
                $formattedLogs[] = $contents;
            }

            // 自定义排序
            if ($sortField !== '__time__') {
                usort($formattedLogs, function($a, $b) use ($sortField, $sort) {
                    // 确保字段存在
                    $valueA = isset($a[$sortField]) ? $a[$sortField] : '';
                    $valueB = isset($b[$sortField]) ? $b[$sortField] : '';

                    // 如果是数字字符串，转换为数字比较
                    if (is_numeric($valueA) && is_numeric($valueB)) {
                        $valueA = (float)$valueA;
                        $valueB = (float)$valueB;
                    }

                    // 根据排序方向比较
                    if ($sort === 'asc') {
                        return $valueA <=> $valueB;
                    }
                    return $valueB <=> $valueA;
                });
            }

            // 应用分页
            $formattedLogs = array_slice($formattedLogs, $offset, $perPage);

            // 返回结果
            return [
                'logs' => $formattedLogs,
                'pagination' => [
                    'total' => $total,
                    'per_page' => $perPage,
                    'current_page' => $page,
                    'last_page' => ceil($total / $perPage),
                    'from' => $offset + 1,
                    'to' => $offset + count($formattedLogs)
                ]
            ];

        } catch (Exception $e) {
            Log::error('SLS Get Error: ' . $e->getMessage());
            throw $e;
        }
    }

    /**
     * 写入日志
     */
    public function putLogs($data)
    {
        try {
            // 确保所有值都是字符串
            $contents = [];
            foreach ($data as $key => $value) {
                $contents[$key] = is_array($value) ? json_encode($value) : (string)$value;
            }

            // 创建日志内容
            $logItem = new Aliyun_Log_Models_LogItem();
            $logItem->setTime(time());
            $logItem->setContents($contents);

            // 创建请求
            $request = new Aliyun_Log_Models_PutLogsRequest(
                $this->project,
                $this->logstore,
                'test_topic',
                '',
                [$logItem]
            );

            // 发送日志
            $response = $this->client->putLogs($request);

            return true;
        } catch (Exception $e) {
            Log::error('SLS Put Error: ' . $e->getMessage());
            throw $e;
        }
    }



    /**
     * 查询所有日志（不分页）
     * @param string $query 查询条件
     * @param int|null $from 开始时间
     * @param int|null $to 结束时间
     * @return array
     * 循环，速度慢，不推荐使用
     */
    public function getAllLogs($query = '', $from = null, $to = null, $sort = 'desc', $sortField = '__time__')
    {
        try {
            $from = $from ?: time() - 3600;
            $to = $to ?: time();

            // 构建查询语句
            $searchQuery = !empty($query) ? $query : '*';
            $allLogs = [];
            $offset = 0;
            $limit = 100;  // 每次获取100条

            do {
                // 创建日志查询请求
                $request = new Aliyun_Log_Models_GetLogsRequest(
                    $this->project,
                    $this->logstore,
                    $from,
                    $to,
                    '',             // topic
                    $searchQuery,   // 查询语句
                    $limit,         // 每次获取数量
                    $offset,        // 当前偏移量
                    $sort === 'desc' // 是否倒序
                );

                // 执行查询
                $response = $this->client->getLogs($request);
                $logs = $response->getLogs();
                $count = count($logs);



                // 处理本批次的日志数据
                foreach ($logs as $log) {
                    $contents = $log->getContents();

                    // 解析 value 字段
                    if (isset($contents['__value__']) && is_string($contents['__value__'])) {
                        try {
                            $decodedValue = json_decode($contents['__value__'], true);
                            if (json_last_error() === JSON_ERROR_NONE) {
                                $contents['__value__'] = $decodedValue;
                            }
                        } catch (\Exception $e) {
                            // 保持原值
                        }
                    }

                    $contents['log_time'] = date('Y-m-d H:i:s', $log->getTime());
                    $allLogs[] = $contents;
                }

                // 更新偏移量
                $offset += $count;

                // 如果返回的数据少于限制数，说明已经没有更多数据
                if ($count < $limit) {
                    break;
                }

            } while (true);

            // 如果需要按其他字段排序
            if ($sortField !== '__time__') {
                usort($allLogs, function($a, $b) use ($sortField, $sort) {
                    $valueA = isset($a[$sortField]) ? $a[$sortField] : '';
                    $valueB = isset($b[$sortField]) ? $b[$sortField] : '';

                    if (is_numeric($valueA) && is_numeric($valueB)) {
                        $valueA = (float)$valueA;
                        $valueB = (float)$valueB;
                    }

                    return $sort === 'asc' ?
                        ($valueA <=> $valueB) :
                        ($valueB <=> $valueA);
                });
            }



            return [
                'logs' => $allLogs,
                'total' => count($allLogs),
                'query_info' => [
                    'query' => $searchQuery,
                    'from' => date('Y-m-d H:i:s', $from),
                    'to' => date('Y-m-d H:i:s', $to),
                    'sort_field' => $sortField,
                    'sort_order' => $sort,
                    'total_batches' => ceil($offset / $limit)
                ]
            ];

        } catch (Exception $e) {
//            Log::error('SLS Get All Logs Error', [
//                'error' => $e->getMessage(),
//                'query' => $searchQuery ?? '',
//                'from' => date('Y-m-d H:i:s', $from),
//                'to' => date('Y-m-d H:i:s', $to),
//                'offset' => $offset ?? 0
//            ]);
            throw $e;
        }
    }


    /**
     * 查询所有日志（不分页）
     * @param $query
     * @param $from
     * @param $to
     * @param $userId
     * @param $sortOrder
     * @return array
     * sql limit的方式，有排序限制，速度快
     */
    public function getAllLogsSql($query = '', $from = null, $to = null, $userId = null, $sortOrder = 'ASC')
    {
        try {
            $from = $from ?: time() - 3600; // 默认查询最近1小时
            $to = $to ?: time();

            // 获取总数，会消耗一定的时间，数量越大，消耗的时间越多，如果想节约时间，可以设置一个最大值类型10W之类的
//            $histogramRequest = new Aliyun_Log_Models_GetHistogramsRequest(
//                $this->project,
//                $this->logstore,
//                $from,
//                $to,
//                '',
//                $query ?: '*'
//            );
//
//            $histogramResponse = $this->client->getHistograms($histogramRequest);
//            $total = $histogramResponse->getTotalCount();
//            $maxResults = $total; // 一次性拉取的最大数量
            $maxResults = 100000; // 一次性拉取的最大数量


            // 构建基础查询语句
            $searchQuery = !empty($query) ? $query : '*';

            // 如果有 user_id 条件，添加筛选
            if ($userId) {
//                $searchQuery .= sprintf(" AND user_id='%s'", $userId);
                $searchQuery .= sprintf(" AND user_id=%d", (int)$userId);

            }



//            Log::info('Starting Query', [
//                'query' => $searchQuery,
//                'from' => date('Y-m-d H:i:s', $from),
//                'to' => date('Y-m-d H:i:s', $to),
//                'sortOrder' => $sortOrder,
//            ]);

            // SQL 查询语句，按时间排序
            $sqlQuery = sprintf(
                "%s | SELECT * ORDER BY __time__ %s LIMIT %d",
                $searchQuery,
                strtoupper($sortOrder), // ASC 或 DESC
                $maxResults
            );

//            Log::info('Executing SQL Query', [
//                'query' => $sqlQuery,
//                'from' => date('Y-m-d H:i:s', $from),
//                'to' => date('Y-m-d H:i:s', $to),
//            ]);

            // 发送请求
            $request = new Aliyun_Log_Models_GetLogsRequest(
                $this->project,
                $this->logstore,
                $from,
                $to,
                '',             // topic
                $sqlQuery,      // 查询语句
                0,              // batchSize 无用
                0,              // offset 无用
                false           // 不自动排序（已通过 SQL 排序）
            );

            $response = $this->client->getLogs($request);
            $logs = $response->getLogs();
            $allLogs = [];

            // 处理返回的日志数据
            foreach ($logs as $log) {
                $contents = $log->getContents();

                // 解析 value 字段
                if (isset($contents['__value__']) && is_string($contents['__value__'])) {
                    try {
                        $decodedValue = json_decode($contents['__value__'], true);
                        if (json_last_error() === JSON_ERROR_NONE) {
                            $contents['__value__'] = $decodedValue;
                        }
                    } catch (\Exception $e) {
                        // 保持原值
                    }
                }

                $contents['log_time'] = date('Y-m-d H:i:s', $log->getTime());
                $allLogs[] = $contents;
            }

//            Log::info('Query Completed', [
//                'fetched' => count($allLogs),
//                'total_expected' => $maxResults,
//            ]);

            return [
                'logs' => $allLogs,
                'total' => count($allLogs),
                'query_info' => [
                    'query' => $searchQuery,
                    'from' => date('Y-m-d H:i:s', $from),
                    'to' => date('Y-m-d H:i:s', $to),
                    'total_fetched' => count($allLogs),
                    'sort_order' => $sortOrder,
                ]
            ];

        } catch (Exception $e) {
//            Log::error('SLS Query Error', [
//                'error' => $e->getMessage(),
//                'query' => $searchQuery ?? '',
//                'from' => date('Y-m-d H:i:s', $from),
//                'to' => date('Y-m-d H:i:s', $to),
//                'sort_order' => $sortOrder,
//            ]);
            throw $e;
        }
    }
}

3、创建config配置

# config/sls.config
<?php
return [
    'aliyun' => [
        'access_key_id' => env('ALIYUN_ACCESS_KEY_ID'),
        'access_key_secret' => env('ALIYUN_ACCESS_KEY_SECRET'),
        'sls' => [
            'endpoint' => env('ALIYUN_SLS_ENDPOINT'),
            'project' => env('ALIYUN_SLS_PROJECT'),
            'logstore' => env('ALIYUN_SLS_LOGSTORE'),
        ],
    ],
];

4、从阿里云获取对应的配置填写在env

# .env,同阿里云账号下的服务器是允许内网访问的
ALIYUN_SLS_ENDPOINT=cn-shenzhen.log.aliyuncs.com
#内网
#ALIYUN_SLS_ENDPOINT=cn-shenzhen-intranet.log.aliyuncs.com
ALIYUN_SLS_PROJECT=
ALIYUN_ACCESS_KEY_ID=
ALIYUN_ACCESS_KEY_SECRET=
ALIYUN_SLS_LOGSTORE=

5、创建控制器进行验证

<?php

namespace App\Http\Controllers\Api;

use App\Http\Controllers\Controller;
use App\Jobs\Sugar;
use App\Services\SLSTimeSeriesService;
use Exception;
use Illuminate\Http\Request;

class MetricsController extends Controller
{
    protected $slsService;

    // 定义允许排序的字段
    // 定义允许排序的字段和默认排序方式
    protected $allowedSortFields = [
        '__time__',      // 日志时间
        '__tag__:__receive_time__', // 接收时间
        'timestamp',     // 自定义时间戳
        'user_id',     // 自定义时间戳
    ];

    public function __construct(SLSTimeSeriesService $slsService)
    {
        $this->slsService = $slsService;
    }


    /**
     * 写入日志
     */
    public function store(Request $request)
    {

        try {
            $data = [
                'event' => 'user_login',
                'user_id' => (string)$request->input('user_id', '1'),
                'ip' => $request->ip(),
                'timestamp' => (string)time(),
                'request_data' => $request->all()
            ];

            $this->slsService->putLogs($data);


            return response()->json([
                'success' => true,
                'message' => 'Log stored successfully'
            ]);
        } catch (Exception $e) {
            return response()->json([
                'success' => false,
                'message' => $e->getMessage()
            ], 500);
        }
    }

    //分页查询
    public function index(Request $request)
    {
        try {
            // 构建查询条件
            $conditions = [];

            if ($request->has('user_id')) {
                $conditions[] = 'user_id = "' . $request->user_id . '"';
            }
            if ($request->has('event')) {
                $conditions[] = 'event = "' . $request->event . '"';
            }

            // 获取排序参数
            $sortField = $request->input('sort_by', '__time__'); // 默认使用日志时间排序
            $sortOrder = $request->input('sort', 'desc');

            // 验证排序字段
            if (!in_array($sortField, $this->allowedSortFields)) {
                $sortField = '__time__';
            }

            // 验证排序方向
            $sortOrder = strtolower($sortOrder) === 'asc' ? 'asc' : 'desc';

            // 构建基本查询
            $query = !empty($conditions) ? implode(' and ', $conditions) : '*';

            // 获取分页参数
            $page = (int)$request->input('page', 1);
            $perPage = (int)$request->input('per_page', 10);

            // 时间范围
            $from = $request->input('from') ? strtotime($request->input('from')) : time() - 3600;
            $to = $request->input('to') ? strtotime($request->input('to')) : time();

            // 记录查询参数


            // 获取数据
            $result = $this->slsService->getLogs(
                $query,
                $from,
                $to,
                $page,
                $perPage,
                $sortOrder,
                $sortField
            );


            return response()->json([
                'success' => true,
                'data' => $result['logs'],
                'pagination' => $result['pagination'],
                'query_info' => [
                    'query' => $query,
                    'conditions' => $conditions,
                    'sort' => [
                        'field' => $sortField,
                        'order' => $sortOrder
                    ],
                    'from' => date('Y-m-d H:i:s', $from),
                    'to' => date('Y-m-d H:i:s', $to)
                ]
            ]);
        } catch (Exception $e) {


            return response()->json([
                'success' => false,
                'message' => $e->getMessage(),
                'query_info' => [
                    'query' => $query ?? '*',
                    'conditions' => $conditions ?? []
                ]
            ], 500);
        }
    }


    /**
     * @param Request $request
     * @return array
     * @throws Exception
     * 循环查询，速度慢，排序无限制，不分页
     */
    public function all(Request $request)
    {
        // 时间范围
        $from = $request->input('from') ? strtotime($request->input('from')) : time() - 3600;
        $to = $request->input('to') ? strtotime($request->input('to')) : time();

        // 自定义排序
        $result = $this->slsService->getAllLogs(
            '*',
            $from,
            $to,
            'desc',
            'timestamp'
        );
        return $result;
    }


    /**
     * @param Request $request
     * @return array
     * @throws Exception
     * sql查询，速度快，排序有限制，需要有索引才能查询出来，不分页
     */
    public function allSql(Request $request)
    {
        $user_id = $request->input('user_id') ?? '';
        // 时间范围
        $from = $request->input('from') ? strtotime($request->input('from')) : time() - 3600;
        $to = $request->input('to') ? strtotime($request->input('to')) : time();


        $result = $this->slsService->getAllLogsSql(
            '*',
            $from,
            $to,
            $user_id,
            'desc'
        );
        return $result;
    }
}

6、效果如下
在这里插入图片描述
在这里插入图片描述


http://www.kler.cn/a/409394.html

相关文章：

  • 《macOS 开发环境配置与应用开发》
  • Charles抓包工具-笔记
  • Redis 常用数据类型插入性能对比：循环插入 vs. 批量插入
  • golang实现TCP服务器与客户端的断线自动重连功能
  • 【计算机网络】网段划分
  • jupyter notebook的 markdown相关技巧
  • 如何快速将Excel数据导入到SQL Server数据库
  • 界面控件DevExpress WPF中文教程：网格视图数据布局的列和卡片字段
  • C++中定义类型名的方法
  • 【Golang】——Gin 框架与数据库集成详解
  • Python的tkinter如何把日志弄进文本框（Text）
  • 大事件管理系统项目总结（上）
  • 【Vscode】不同系统快捷键
  • 论防火墙对网络安全的重要性
  • 【大数据学习 | Spark-Core】Spark提交及运行流程
  • Oracle 执行计划查看方法汇总及优劣对比
  • 信息收集ip测活-Python脚本编写
  • Java零拷贝一步曲——Linux 中的零拷贝技术
  • C++ Qt 识别U盘/串口
  • 传输控制协议（TCP）和用户数据报协议（UDP）
  • ODBC连接PostgreSQL数据库后，网卡DOWN后，客户端进程阻塞问题解决方法
  • 无法加载文件 C:\dev\nodejs\cnpm.ps1，因为在此系统上禁止运行脚本。问题解决
  • 用java和redis实现考试成绩排行榜
  • RabbitMQ 之 死信队列
  • 基于Python Web的社区爱心养老管理系统设计与实现
  • Mysql的加锁情况详解