Проблема с udp сокетами

Делаю клиент-серверную игру на 2х игроков. Сервер ждет пока 2 игрока пришлют информацию и затем рассылает друг друг. Первый клиент отправляет инфу, сервер ее выводит, второй клиент отправляет, сервер выводит ее и она такая же как от первого. Потом сервер отправляет каждому инфу и клиенты получают ту же инфу что и отправляли сами. Как решить эту проблему?

Код сервера:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using CourseWork;
using System.Net;
using System.Threading;
using System.Net.Sockets;

namespace CourseWorkServer
{
    public class Program
    {
        public static Socket ServerSocket;
        public static EndPoint ServerIP;
        //public static Socket Player1Socket;
        //public static Socket Player2Socket;
        public static EndPoint Player1IP;
        public static EndPoint Player2IP;
        public static EndPoint tempIP = new IPEndPoint(IPAddress.Any, 0);
        public static byte[] Player1InitInfo;
        public static byte[] Player2InitInfo;
        public static byte[] Player1KBS = new byte[100000];
        public static byte[] Player2KBS = new byte[100000];
        public static Thread Player1Thread;
        public static Thread Player2Thread;
        public static byte[] TempData = new byte[10000];
        public static void InitInfo()
        {
            while (true)
            {
                tempIP = new IPEndPoint(IPAddress.Any, 0);
                Console.WriteLine("Waiting for connections...");
                if (Player1InitInfo == null)
                {
                    var len = ServerSocket.ReceiveFrom(TempData,ref tempIP);
                    Console.WriteLine(tempIP.ToString());
                    if (len != 0)
                    {
                        Player1IP = tempIP;
                        var msg = (NetMessage)DataSerializer.Deserialize(TempData);
                        if (msg.PacketType == PacketType.INIT_INFO)
                        {
                            Player1InitInfo = DataSerializer.Serialize(msg.Data);
                            Console.WriteLine(msg.Data.ToString());
                            Console.WriteLine("Received init info from Player 1");
                        }
                    }
                }
                else if (Player2InitInfo == null)
                {
                    TempData = null;
                    TempData = new byte[10000];
                    var len = ServerSocket.ReceiveFrom(TempData, ref tempIP);
                    Console.WriteLine(tempIP.ToString());
                    if (len != 0)
                    {
                        Player2IP = tempIP;
                        var msg = (NetMessage)DataSerializer.Deserialize(TempData);
                        if (msg.PacketType == PacketType.INIT_INFO)
                        {
                            Player2InitInfo = DataSerializer.Serialize(msg.Data);
                            Console.WriteLine(msg.Data.ToString());
                            Console.WriteLine("Received init info from Player 2");
                        }
                    }
                }
                if (Player1InitInfo != null && Player2InitInfo != null)
                {
                    ServerSocket.SendTo(DataSerializer.Serialize(new NetMessage(PacketType.INIT_INFO, DataSerializer.Deserialize(Player1InitInfo))), Player2IP);
                    ServerSocket.SendTo(DataSerializer.Serialize(new NetMessage(PacketType.INIT_INFO, DataSerializer.Deserialize(Player2InitInfo))), Player1IP);
                    Console.WriteLine("Send init info to both players");
                    return;
                }
            }
        }
        public static void Main(String[] args)
        {
            ServerSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
            ServerIP = new IPEndPoint(IPAddress.Loopback, 8080);
            Console.Title = ServerIP.ToString();
            ServerSocket.Bind(ServerIP);
            InitInfo();
            Player1Thread = new Thread(new ThreadStart(ProccessClient1));
            Player2Thread = new Thread(new ThreadStart(ProccessClient2));
            Player1Thread.Start();
            Player2Thread.Start();
            while (true) {
                Thread.Sleep(16);
            }
        }
        public static void ProccessClient1()
        {
            Object locker = new object();
            lock (locker)
            {
                ServerSocket.ReceiveFrom(Player1KBS, ref tempIP);
                if (tempIP == Player1IP)
                {
                    Console.WriteLine("Received update from Player 1");
                    ServerSocket.SendTo(Player1KBS, Player2IP);
                    Console.WriteLine("Sent update to Player 2");
                }
            }
        }
        public static void ProccessClient2()
        {
            Object locker = new object();
            lock (locker)
            {
                ServerSocket.ReceiveFrom(Player2KBS, ref tempIP);
                if (tempIP == Player2IP)
                {
                    Console.WriteLine("Received update from Player 2");
                    ServerSocket.SendTo(Player2KBS, Player1IP);
                    Console.WriteLine("Sent update to Player 1");
                }
            }
        }
    }
}
Код клиента
    using Microsoft.Xna.Framework;
    using Microsoft.Xna.Framework.Graphics;
    using Microsoft.Xna.Framework.Input;
    using Microsoft.Xna.Framework.Media;
    using Microsoft.Xna.Framework.Audio;
    using System.Collections.Generic;
    using Lidgren.Network;
    using System;
    using System.Net;
    using System.Net.Sockets;
    using System.Text;
    using System.Threading;
    namespace CourseWork
    {
        public class NarutoFighting : Game
        {
            GraphicsDeviceManager graphics;
            SpriteBatch spriteBatch;
            public Texture2D Background;
            public Vector2 BackgroundPosition;
            public Song BackgroundTheme;
            public Character Player1;
            public Character Player2;
            public String Player1Event;
            public String Player2Event;
            public PlayerStartInfo Player1Info = new PlayerStartInfo(Characters.NARUTO, 100, 100);
            public PlayerStartInfo Player2Info;
            public byte[] DataTemp = new Byte[100000];
            public Socket ClientSocket;
            public EndPoint ClientIP;
            public EndPoint ServerIP;
            public bool BeganConnection;
            public bool SentInitInfo;
            public bool ReceivedInitInfo;
            public Characters charax;
            public int x, y;
            Input Player1Input = new Input()
            {
                Up = Keys.W,
                Left = Keys.A,
                Down = Keys.S,
                Right = Keys.D,
                LightHit = Keys.J,
                HardHit = Keys.K,
                Kick = Keys.L,
                Block = Keys.B,
                Special1 = Keys.D1,
                Special2 = Keys.D2,
                Special3 = Keys.D3,
            };
            Input Player2Input = new Input()
            {
                Up = Keys.Up,
                Left = Keys.Left,
                Down = Keys.Down,
                Right = Keys.Right,
                LightHit = Keys.NumPad4,
                HardHit = Keys.NumPad5,
                Kick = Keys.NumPad6,
                Block = Keys.NumPad8,
                Special1 = Keys.NumPad1,
                Special2 = Keys.NumPad2,
                Special3 = Keys.NumPad3,
            };
            public Thread NetworkThread;

            public NarutoFighting(String playerChar = "ITACHI", int x = 100, int y = 400, int port = 20000)
            {
                graphics = new GraphicsDeviceManager(this);
                Content.RootDirectory = "Content";
                ClientSocket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
                ClientIP = new IPEndPoint(IPAddress.Loopback, port);
                ServerIP = new IPEndPoint(IPAddress.Loopback, 8080);
                ClientSocket.Bind(ClientIP);
                ClientSocket.Connect(ServerIP);
                Player1Info.PlayerChar = (Characters)Enum.Parse(typeof(Characters), playerChar);
                Player1Info.PlayerXPos = x;
                Player1Info.PlayerYPos = y;
            }
            //public void GetConnection()
            //{
            //    Window.Title = ClientSocket.LocalEndPoint.ToString();
            //    NetMessage msg = new NetMessage(PacketType.CONNECT, ClientSocket.LocalEndPoint.ToString());
            //    var bytes = DataSerializer.Serialize(msg);
            //    ClientSocket.Send(bytes);
            //    BeganConnection = true;
            //    DataTemp = Encoding.UTF8.GetBytes(ClientIP.ToString());
            //    ClientSocket.Send(DataTemp);
            //    BeganConnection = true;
            //}
            public void SendInitInfo()
            {
                NetMessage msg = new NetMessage(PacketType.INIT_INFO, Player1Info);
                ClientSocket.Send(DataSerializer.Serialize(msg));
                SentInitInfo = true;
            }
            public void GetInitInfo()
            {
                var len = ClientSocket.Receive(DataTemp);
                if (len != 0)
                {
                    var data = (NetMessage)DataSerializer.Deserialize(DataTemp);
                    if (data.PacketType == PacketType.INIT_INFO)
                    {
                        var temp = (NetMessage)DataSerializer.Deserialize(DataTemp);
                        if (Player1Info != (PlayerStartInfo)temp.Data)
                        {
                            Player2Info = (PlayerStartInfo)temp.Data;
                            Player2 = CharacterLoader.InitCharacter(Player2Info.PlayerChar, Player2Info.PlayerXPos, Player2Info.PlayerYPos);
                            Player2.GameLoop = this;
                            Player2.LoadUI();
                            Player2.LoadAnimations();
                            Player2.LoadEffects();
                            Player2.LoadSounds();
                            Player2.CurrentAnimation = Player2.Position.X < GraphicsDevice.Adapter.CurrentDisplayMode.Width / 2 ? Player2.Animations["idle_right"] : Player2.Animations["idle_left"];
                            CollisionManager.PlayersPool.Add(Player2);
                            ReceivedInitInfo = true;
                        }
                    }
                }
            }

            protected override void Initialize()
            {
                Input Player1Input = new Input()
                {
                    Up = Keys.W,
                    Left = Keys.A,
                    Down = Keys.S,
                    Right = Keys.D,
                    LightHit = Keys.J,
                    HardHit = Keys.K,
                    Kick = Keys.L,
                    Block = Keys.B,
                    Special1 = Keys.D1,
                    Special2 = Keys.D2,
                    Special3 = Keys.D3,
                };
                Input Player2Input = new Input()
                {
                    Up = Keys.Up,
                    Left = Keys.Left,
                    Down = Keys.Down,
                    Right = Keys.Right,
                    LightHit = Keys.NumPad4,
                    HardHit = Keys.NumPad5,
                    Kick = Keys.NumPad6,
                    Block = Keys.NumPad8,
                    Special1 = Keys.NumPad1,
                    Special2 = Keys.NumPad2,
                    Special3 = Keys.NumPad3,
                };
                Player1 = CharacterLoader.InitCharacter(Player1Info.PlayerChar, Player1Info.PlayerXPos, Player1Info.PlayerYPos);
                Player1.GameLoop = this;
                Player1.LoadUI();
                Player1.LoadAnimations();
                Player1.LoadEffects();
                Player1.LoadSounds();
                Player1.CurrentAnimation = Player1.Position.X < GraphicsDevice.Adapter.CurrentDisplayMode.Width / 2 ? Player1.Animations["idle_right"] : Player1.Animations["idle_left"];
                Player1.InputConfig = Player1Input;
                //Player2.InputConfig = Player2Input;

                CollisionManager.PlayersPool.Add(Player1);
                Background = Content.Load<Texture2D>("Backgrounds/background");
                //BackgroundTheme = Content.Load<Song>("Sound/Music/16 - need to be strong");
                //MediaPlayer.Play(BackgroundTheme);
                BackgroundPosition = new Vector2(0, 0);
                base.Initialize();
                NetworkThread = new Thread(new ThreadStart(NetWorkUpdate));
                NetworkThread.Start();
            }
            protected override void LoadContent()
            {
                spriteBatch = new SpriteBatch(GraphicsDevice);
            }
            protected override void UnloadContent()
            {
            }
            protected override void Dispose(bool disposing)
            {
                base.Dispose(disposing);
                ClientSocket.Close();
            }
            public void NetWorkUpdate()
            {
                while (true)
                {
                    if (!SentInitInfo)
                    {
                        SendInitInfo();
                    }
                    else if (!ReceivedInitInfo)
                    {
                        GetInitInfo();
                    }
                    else if (Player2 != null)
                    {
                        DataTemp = Encoding.UTF8.GetBytes(Player1.SendPackage());
                        ClientSocket.Send(DataTemp);
                        var len = ClientSocket.Receive(DataTemp);
                        Player2.Event = Encoding.UTF8.GetString(DataTemp, 0, len);
                    }
                }
            }
            protected override void Update(GameTime gameTime)
            {
                CollisionManager.ManageCollisions();
                Player1.SetEvent();
                Player1.Update(gameTime);
                //NetWorkUpdate();
                if (Player2 != null)
                {
                    Player2.Update(gameTime);
                }
                //Window.Title = "X=" + Player1.Position.X + " Y=" + Player1.Position.Y + " EndPoint: " + ClientIP + " BC: " + BeganConnection + " SII: " + SentInitInfo + " RII: " + ReceivedInitInfo;
                base.Update(gameTime);
            }
            protected override void Draw(GameTime gameTime)
            {
                GraphicsDevice.Clear(Color.White);
                // TODO: Add your drawing code here
                spriteBatch.Begin();
                spriteBatch.Draw(Background, BackgroundPosition, Color.White);
                Player1.Draw(spriteBatch, gameTime);
                if (Player2 != null)
                {
                    Player2.Draw(spriteBatch, gameTime);
                }
                spriteBatch.End();
                base.Draw(gameTime);
            }
        }
    }

Код DataSerializer

    using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Formatters.Binary;
using System.Text;
using System.Threading.Tasks;
namespace CourseWork
{
    public class DataSerializer
    {
        public static BinaryFormatter binaryFormatter = new BinaryFormatter();
        public static MemoryStream memoryStream = new MemoryStream();
        private DataSerializer() { }
        public static byte[] Serialize(Object dataPackage)
        {
            binaryFormatter.Serialize(memoryStream, dataPackage);
            return memoryStream.GetBuffer();
        }
        public static Object Deserialize(byte[] temp)
        {
            memoryStream.Write(temp, 0, temp.Length);
            memoryStream.Seek(0, SeekOrigin.Begin);
            return (Object)binaryFormatter.Deserialize(memoryStream);
        }
    }
}