主页 / coding / 问题 / 78926230
Accepted
lijiang99
Asked: 2024-08-29 13:30:50 +0800 CST

使用 C++ 进行多线程磁盘扫描时出现访问冲突错误

  • 772

我正在尝试使用多线程扫描系统磁盘并构建包含所有文件路径的 Trie 树。但是在测试过程中我遇到了一个问题。

在我的 main.cpp 中，当循环索引 i 为 2、4 或 6 时，程序运行正常。但是，当 i 为 8 或 10 时（有时 i = 8 有效），程序会在 WorkStealQueue::pop 函数中的 if (m_tasks.empty()) return false; 行处崩溃，并出现访问冲突错误，并显示消息“读取位置 0x00000014 时发生访问冲突”。

我尝试了各种方法来解决此问题，但均未成功。有任何帮助或指导可以解决此问题吗？

下面是详细的代码实现（我的操作平台是Windows）：

// workstealthreadpool.h
#pragma once

#include <deque>
#include <functional>
#include <mutex>
#include <thread>
#include <future>
#include <vector>

class WorkStealQueue
{
public:
    WorkStealQueue() = default;
    WorkStealQueue(const WorkStealQueue& rhs) = delete;
    WorkStealQueue& operator=(const WorkStealQueue& rhs) = delete;
    ~WorkStealQueue() = default;

    void push(std::function<void()> task);
    bool pop(std::function<void()>& task);
    bool steal(std::function<void()>& task);

private:
    std::deque<std::function<void()>> m_tasks;
    std::mutex m_mutex;
};

class WorkStealThreadPool
{
public:
    explicit WorkStealThreadPool(std::size_t threadNums)
        : m_stop(false) { init(threadNums); }
    ~WorkStealThreadPool();

    template<typename Callback, typename... Args>
    auto addTask(Callback&& func, Args&&... args)->std::future<typename std::result_of<Callback(Args...)>::type>;

private:
    void init(std::size_t threadNums);
    bool stealTask(std::function<void()>& task);
    void worker(size_t index);

private:
    std::vector<std::thread> m_workThreads;
    std::vector<std::unique_ptr<WorkStealQueue>> m_taskQueues;
    std::atomic<bool> m_stop;
    static thread_local std::size_t m_index;
};

template<typename Callback, typename... Args>
auto WorkStealThreadPool::addTask(Callback&& func, Args&&... args) -> std::future<typename std::result_of<Callback(Args...)>::type>
{
    using returnType = typename std::result_of<Callback(Args...)>::type;
    auto task = std::make_shared<std::packaged_task<returnType()>>(std::bind(std::forward<Callback>(func), std::forward<Args>(args)...));
    std::future<returnType> result = task->get_future();
    {
        m_taskQueues[m_index]->push([task]() { (*task)(); });
    }
    return result;
}

// workstealthreadpool.cpp
#include "workstealthreadpool.h"

void WorkStealQueue::push(std::function<void()> task)
{
    std::lock_guard<std::mutex> lock(m_mutex);
    m_tasks.emplace_back(std::move(task));
}

bool WorkStealQueue::pop(std::function<void()>& task)
{
    std::lock_guard<std::mutex> lock(m_mutex);
    if (m_tasks.empty()) return false;
    task = std::move(m_tasks.front());
    m_tasks.pop_front();
    return true;
}

bool WorkStealQueue::steal(std::function<void()>& task)
{
    std::lock_guard<std::mutex> lock(m_mutex);
    if (m_tasks.empty()) return false;
    task = std::move(m_tasks.back());
    m_tasks.pop_back();
    return true;
}

thread_local std::size_t WorkStealThreadPool::m_index;

void WorkStealThreadPool::init(std::size_t threadNums)
{
    for (std::size_t i = 0; i < threadNums; ++i)
    {
        m_taskQueues.emplace_back(std::make_unique<WorkStealQueue>());
        m_workThreads.emplace_back(&WorkStealThreadPool::worker, this, i);
    }
}

WorkStealThreadPool::~WorkStealThreadPool()
{
    m_stop = true;
    for (std::thread& workerThread : m_workThreads)
    {
        if (workerThread.joinable())
        {
            workerThread.join();
        }
    }
}

bool WorkStealThreadPool::stealTask(std::function<void()>& task)
{
    for (std::size_t i = 0; i < m_taskQueues.size(); ++i)
    {
        std::size_t index = (m_index + i + 1) % m_taskQueues.size();
        if (m_taskQueues[index]->steal(task))
        {
            return true;
        }
    }
    return false;
}

void WorkStealThreadPool::worker(std::size_t index)
{
    m_index = index;
    while (!m_stop)
    {
        std::function<void()> task;
        if (m_taskQueues[m_index]->pop(task) || stealTask(task))
        {
            task();
        }
        else
        {
            std::this_thread::yield();
        }
    }
}

// scanner.h
#pragma once

#include "workstealthreadpool.h"
#include <QMap>
#include <QString>

struct TrieNode
{
    QMap<TrieNode*, QString> childs;
    TrieNode* parent = nullptr;
};

class Scanner
{
public:
    explicit Scanner(std::size_t threadNums);
    virtual ~Scanner();
    virtual void scanDrives(const QStringList& drives);
    virtual bool isScanCompleted();
    virtual std::vector<TrieNode*> fetchScanResults();

private:
    void scanCore(const QString& currentPath, TrieNode* parent);
    void clearTrie(TrieNode* root);

private:
    TrieNode* m_root;
    WorkStealThreadPool* m_threadPool;
    std::mutex m_mutex;
    std::vector<TrieNode*> m_fileNodes;
    std::atomic<int> m_taskCount;
};

#ifdef _DEBUG
void Print(TrieNode* root);
void Print(std::vector<TrieNode*>* fileNodes);
#endif

// scanner.cpp
#include "scanner.h"
#include <QDir>

Scanner::Scanner(std::size_t threadNums)
    : m_root(nullptr)
    , m_threadPool(nullptr)
    , m_taskCount(0)
{
    if (threadNums != 0)
    {
        m_threadPool = new WorkStealThreadPool(threadNums);
    }
}

Scanner::~Scanner()
{
    delete m_threadPool;
    clearTrie(m_root);
}

void Scanner::scanDrives(const QStringList& drives)
{
    clearTrie(m_root);
    m_fileNodes.clear();
    m_root = new TrieNode();
    for (const QString& drive : drives)
    {
        TrieNode* child = new TrieNode();
        child->parent = m_root;
        m_root->childs[child] = drive;
        scanCore(drive, child);
    }
}

bool Scanner::isScanCompleted()
{
    return m_taskCount.load(std::memory_order_acquire) == 0;
}

std::vector<TrieNode*> Scanner::fetchScanResults()
{
    if (!isScanCompleted())
    {
        std::lock_guard<std::mutex> lock(m_mutex);
        return m_fileNodes;
    }
    return m_fileNodes;
}

void Scanner::scanCore(const QString& currentPath, TrieNode* parent)
{
    QDir dir(currentPath);
    if (!dir.exists()) return;

    QStringList fileNames = dir.entryList(QDir::Files);
    for (const QString& fileName : fileNames)
    {
        TrieNode* child = new TrieNode();
        child->parent = parent;
        parent->childs[child] = fileName;

        std::lock_guard<std::mutex> lock(m_mutex);
        m_fileNodes.emplace_back(child);
    }

    QStringList subdirNames = dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot);
    for (const QString& subdirName : subdirNames)
    {
        QString childPath = currentPath + QDir::separator() + subdirName;
        TrieNode* child = new TrieNode();
        child->parent = parent;
        parent->childs[child] = subdirName + "/";

        if (m_threadPool)
        {
            m_taskCount.fetch_add(1, std::memory_order_release);
            m_threadPool->addTask([this, childPath, child]
                {
                    scanCore(childPath, child);
                    m_taskCount.fetch_sub(1, std::memory_order_acquire);
                }
            );
        }
        else {
            scanCore(childPath, child);
        }
    }

}

void Scanner::clearTrie(TrieNode* root)
{
    if (root == nullptr || root->childs.empty()) return;
    for (auto iter = root->childs.begin(); iter != root->childs.end(); ++iter)
    {
        clearTrie(iter.key());
    }
    delete root;
}

#ifdef _DEBUG
void Print(TrieNode* root)
{
    static int level = 0;
    if (root == nullptr || root->childs.empty()) return;
    for (auto iter = root->childs.begin(); iter != root->childs.end(); ++iter)
    {
        qDebug().noquote() << QString("    ").repeated(level) << iter.value();
        ++level;
        Print(iter.key());
        --level;
    }
}
#endif

#ifdef _DEBUG
void Print(std::vector<TrieNode*>* fileNodes)
{
    for (TrieNode* fileNode : *fileNodes)
    {
        qDebug().noquote() << fileNode->parent->childs[fileNode];
    }
}
#endif

//main.cpp
#include "scanner.h"
#include <QStorageInfo>
#include <chrono>

void TestScanDrives(Scanner& scanner, const QStringList& drives)
{
    scanner.scanDrives(drives);
    while (!scanner.isScanCompleted())
    {
        std::this_thread::sleep_for(std::chrono::seconds(1));
    }
}

int main()
{
    QStringList drives;
    for (const QStorageInfo& drive : QStorageInfo::mountedVolumes())
    {
        if (drive.isValid() && drive.isReady())
        {
            drives << drive.rootPath();
        }
    }

    for (int i = 2; i <= 10; i += 2)
    {
        Scanner scanner(i);
        TestScanDrives(scanner, drives);
    }
    
    return 0;
}

我仔细检查了 WorkStealQueue 类的实现，特别是 pop 和 push 方法，以确保它们正确处理任务。我将原子变量 m_taskCount 的内存顺序调整为不同的值，以查看它是否影响了该问题。我尝试改进 WorkStealThreadPool 中的线程同步和任务终止逻辑，但没有看到明显的改进。

c++

1 个回答

  1. Best Answer

    您拥有非同步的访问和修改m_taskQueues.size()权m_taskQueues[index]

    for (std::size_t i = 0; i < threadNums; ++i)
    {
        m_taskQueues.emplace_back(std::make_unique<WorkStealQueue>());
        m_workThreads.emplace_back(&WorkStealThreadPool::worker, this, i);
    }

    当一个工作线程产生时，它可以尝试从它之后的线程中窃取工作，并且m_taskQueues.size()不会与之同步m_taskQueues[index]（因为没有互斥锁m_taskQueues，并且另一个线程可能会看到大小增加但看不到向量数据被修改，因为这是一场竞争，也可能看不到队列完全构造因为那些不是原子指针），所以将这个循环分成两个。

    for (std::size_t i = 0; i < threadNums; ++i)
    {
        m_taskQueues.emplace_back(std::make_unique<WorkStealQueue>());
    }
for (std::size_t i = 0; i < threadNums; ++i)
    {
        m_workThreads.emplace_back(&WorkStealThreadPool::worker, this, i);
    }

    产生一个线程是一个同步点，当它发生时，所有队列都将完全构建，因此不会发生竞争m_taskQueues.size()。

    • 1

相关问题